diff --git a/packages/serve-sim/Sources/SimNative/CaptureEngine.swift b/packages/serve-sim/Sources/SimNative/CaptureEngine.swift new file mode 100644 index 000000000..fa541d89f --- /dev/null +++ b/packages/serve-sim/Sources/SimNative/CaptureEngine.swift @@ -0,0 +1,219 @@ +import Foundation +import CoreVideo +import CoreMedia +import os + +// The capture + encode engine, reused verbatim from SimStreamHelper. Replicates +// main.swift's frameHandler: MJPEG always encodes while clients exist; H.264 runs +// only while AVCC is active. Encoded bytes (JPEG, or natively-framed AVCC +// envelopes) are handed back through a Swift closure on a native encode thread; +// the node-swift binding (sim-module.swift) marshals them onto the JS thread via +// a NodeAsyncQueue (threadsafe function). + +struct Frame: Identifiable { + let id = UUID() + let pixelBuffer: CVPixelBuffer +} + +protocol FrameEncoder { + associatedtype Encoded + func encode(_ frame: Frame) async throws -> Encoded +} + +protocol CaptureConsuming: Sendable { + // this is intentionally synchronous. CaptureEngine sends all frames to all consumers, + // and lets them handle internal backpressure as they see fit. if instead this were async + // (and CaptureEngine waited for all consumers to finish), a single bad consumer could + // jam up the entire pipeline. + func handleFrame(_ frame: Frame) +} + +actor CaptureConsumer: CaptureConsuming { + nonisolated let continuation: AsyncStream.Continuation + + init( + encoder: E, + onFrame: @escaping @isolated(any) (E.Encoded) async -> Void + ) { + let (stream, continuation) = AsyncStream.makeStream( + of: Frame.self, + // drop old frames if there's backpressure + bufferingPolicy: .bufferingNewest(1) + ) + self.continuation = continuation + Task { + _ = onFrame.isolation + for await frame in stream { + do { + let encoded = try await encoder.encode(frame) + await onFrame(encoded) + } catch { + print("error encoding frame: \(error)") + continue + } + } + } + } + + nonisolated func handleFrame(_ frame: Frame) { + continuation.yield(frame) + } + + deinit { continuation.finish() } +} + +actor CaptureEngine { + private enum Phase { + case unstarted + case starting + case running + case stopped + } + + private let deviceUDID: String + private let frameCapture = FrameCapture() + private var phase = Phase.unstarted + + // mjpeg is stateless so we can share a single encoder instance + private let mjpegEncoder = MJPEGEncoder() + + private(set) var screenSize = Dimensions(width: 0, height: 0) + private var consumers = [UUID: CaptureConsuming]() + + init(deviceUDID: String) { + self.deviceUDID = deviceUDID + } + + func start() async throws { + guard phase == .unstarted else { return } + phase = .starting + // Latch `started` only after capture actually begins: if start() throws + // (e.g. device not booted), a later retry should still be allowed. + let (frames, frameContinuation) = AsyncStream.makeStream( + of: Frame.self, + // drop old frames if there's backpressure + bufferingPolicy: .bufferingNewest(1) + ) + try await frameCapture.start(deviceUDID: deviceUDID) { pixelBuffer, _ in + frameContinuation.yield(Frame(pixelBuffer: pixelBuffer)) + } + Task { + for await frame in frames { + handleFrame(frame) + } + } + phase = .running + } + + private func addConsumer( + encoder: E, + onFrame: sending @escaping @isolated(any) (E.Encoded) async -> Void + ) -> (@Sendable () async -> Void) { + let consumer = CaptureConsumer(encoder: encoder) { [weak self] encoded in + guard let self, await self.phase == .running else { return } + await onFrame(encoded) + } + let id = UUID() + consumers[id] = consumer + return { await self.removeConsumer(id) } + } + + private func removeConsumer( + _ id: UUID + ) { + consumers.removeValue(forKey: id) + } + + private func handleFrame(_ frame: Frame) { + guard phase == .running else { return } + screenSize = frame.pixelBuffer.dimensions + for consumer in consumers.values { + consumer.handleFrame(frame) + } + } + + func addMJPEGConsumer( + onFrame: sending @escaping (Dimensions, Data) async -> Void + ) -> (@Sendable () async -> Void) { + return addConsumer(encoder: mjpegEncoder, onFrame: { [weak self] data in + guard let self else { return } + await onFrame(screenSize, data) + }) + } + + func addAVCCConsumer( + onFrame: sending @escaping (Dimensions, Data, Int32) async -> Void + ) -> (@Sendable () async -> Void) { + addConsumer(encoder: AVCCEncoder()) { [weak self] encoded in + let flagDescription: Int32 = 1 << 0 + let flagKeyframe: Int32 = 1 << 1 + + guard let self else { return } + if let description = encoded.description { + await onFrame( + screenSize, + AVCCEnvelope.description(avcc: description), + flagDescription, + ) + } + switch encoded.kind { + case .keyframe: + await onFrame( + screenSize, + AVCCEnvelope.keyframe(avcc: encoded.avcc), + flagKeyframe, + ) + case .delta: + await onFrame( + screenSize, + AVCCEnvelope.delta(avcc: encoded.avcc), + 0, + ) + } + } + } + + func stop() { + if phase == .stopped { return } + phase = .stopped + Task { [frameCapture] in await frameCapture.stop() } + consumers.removeAll() + } +} + +actor MJPEGEncoder: FrameEncoder { + private let videoEncoder = VideoEncoder(quality: 0.7) + private var lastImage: (UUID, Data)? + + init() {} + + func encode(_ frame: Frame) async throws -> Data { + if let (id, data) = lastImage, id == frame.id { return data } + let data = try await videoEncoder.encode(pixelBuffer: frame.pixelBuffer) + lastImage = (frame.id, data) + return data + } +} + +actor AVCCEncoder: FrameEncoder { + private static let timeout: Duration = .milliseconds(500) + + let h264Encoder = H264Encoder(fps: 60) + var forceKeyframe = true + + init() {} + + func encode(_ frame: Frame) async throws -> H264Encoder.Encoded { + // TODO: cancel after timeout using TaskGroup + let result = try await h264Encoder.encode( + frame.pixelBuffer, + forceKeyframe: forceKeyframe, + ) + forceKeyframe = false + return result + } + + deinit { + Task { [h264Encoder] in await h264Encoder.stop() } + } +} diff --git a/packages/serve-sim/Sources/SimNative/FrameCapture.swift b/packages/serve-sim/Sources/SimNative/FrameCapture.swift index 423c5cda9..1a677e6b3 100644 --- a/packages/serve-sim/Sources/SimNative/FrameCapture.swift +++ b/packages/serve-sim/Sources/SimNative/FrameCapture.swift @@ -12,14 +12,17 @@ import ObjectiveC /// for late-joining clients. /// /// Pipeline: IOSurface (shared memory) → CVPixelBuffer (zero-copy) → H.264 encode -final class FrameCapture { +actor FrameCapture { + private let queue = DispatchSerialQueue(label: "frame-capture", qos: .userInteractive) + nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() } + + private var photocopier = Photocopier() private var onFrame: ((CVPixelBuffer, CMTime) -> Void)? private var frameCount: UInt64 = 0 private(set) var capturedWidth: Int = 0 private(set) var capturedHeight: Int = 0 - private var idleTimer: DispatchSourceTimer? - private let captureQueue = DispatchQueue(label: "frame-capture", qos: .userInteractive) - private var lastCaptureTimeMs: UInt64 = 0 + private var idleTimer: Task? + private var lastCaptureTime: ContinuousClock.Instant = .now private var lastSeeds: [ObjectIdentifier: UInt32] = [:] private var rewireTickCount: Int = 0 /// Interval at which the idle timer re-emits the current frame even when @@ -32,13 +35,13 @@ final class FrameCapture { /// one subscriber is due for it — a late-joining relay subscriber on an /// idle sim never gets a cached frame to show. /// Re-emitting at ~5 fps fixes both without meaningful CPU cost. - private static let idleIntervalMs: UInt64 = 200 + private static let idleInterval: ContinuousClock.Duration = .milliseconds(200) private var descriptors: [NSObject] = [] - private var callbackUUIDs: [ObjectIdentifier: NSUUID] = [:] + private var callbackUUIDs: [ObjectIdentifier: UUID] = [:] private var ioClient: NSObject? - func start(deviceUDID: String, onFrame: @escaping (CVPixelBuffer, CMTime) -> Void) throws { + func start(deviceUDID: String, onFrame: @escaping @Sendable (CVPixelBuffer, CMTime) -> Void) throws { self.onFrame = onFrame SimFrameworks.load() @@ -156,70 +159,56 @@ final class FrameCapture { // MARK: - Frame callbacks via objc_msgSend - private func registerFrameCallbacks(desc: NSObject) throws { - let regSel = NSSelectorFromString("registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:") + private func registerFrameCallbacks(desc: AnyObject) throws { + let regSel = #selector(FramebufferDescriptor.registerScreenCallbacks) guard desc.responds(to: regSel) else { throw makeError(8, "Descriptor doesn't support registerScreenCallbacks") } - guard let msgSendPtr = dlsym(UnsafeMutableRawPointer(bitPattern: -2), "objc_msgSend") else { - throw makeError(9, "objc_msgSend not found") - } - - typealias MsgSendFunc = @convention(c) ( - AnyObject, Selector, AnyObject, AnyObject, AnyObject, AnyObject, AnyObject - ) -> Void - let msgSend = unsafeBitCast(msgSendPtr, to: MsgSendFunc.self) - - let uuid = NSUUID() + let uuid = UUID() callbackUUIDs[ObjectIdentifier(desc)] = uuid - let frameCallback: @convention(block) () -> Void = { [weak self] in - self?.captureQueue.async { self?.captureFrame() } - } - let surfacesCallback: @convention(block) () -> Void = { [weak self] in - self?.captureQueue.async { self?.captureFrame() } - } - let propsCallback: @convention(block) () -> Void = {} - - msgSend( - desc, regSel, - uuid, captureQueue as AnyObject, - frameCallback as AnyObject, surfacesCallback as AnyObject, propsCallback as AnyObject + desc.registerScreenCallbacks( + uuid: uuid, + callbackQueue: queue, + frameCallback: { [self] in assumeIsolated { $0.captureFrame() } }, + surfacesChangedCallback: { [self] in assumeIsolated { $0.captureFrame() } }, + propertiesChangedCallback: {} ) } private func startIdleTimer() { - let timer = DispatchSource.makeTimerSource(queue: captureQueue) - timer.schedule(deadline: .now().advanced(by: .milliseconds(Int(Self.idleIntervalMs))), - repeating: .milliseconds(Int(Self.idleIntervalMs))) - timer.setEventHandler { [weak self] in - guard let self else { return } - let nowMs = DispatchTime.now().uptimeNanoseconds / 1_000_000 - if (nowMs - self.lastCaptureTimeMs) >= Self.idleIntervalMs { - self.captureFrame() + self.idleTimer = Task { [weak self] in + while !Task.isCancelled { + guard let self else { return } + await self.onIdleTimerTick() + try? await Task.sleep(for: Self.idleInterval) } - // Self-heal: if we've never captured a frame, the cached descriptor - // is likely stale. Re-wire the pipeline periodically (every ~1s) - // until frames start flowing. - if self.frameCount == 0 { - self.rewireTickCount += 1 - if self.rewireTickCount % 5 == 0 { - do { - try self.wireUpFramebuffer() - } catch { - // Swallow — we'll try again on the next tick. - } + } + } + + private func onIdleTimerTick() { + let now = ContinuousClock.now + guard (now - self.lastCaptureTime) >= Self.idleInterval else { return } + self.captureFrame(force: true) + // Self-heal: if we've never captured a frame, the cached descriptor + // is likely stale. Re-wire the pipeline periodically (every ~1s) + // until frames start flowing. + if self.frameCount == 0 { + self.rewireTickCount += 1 + if self.rewireTickCount % 5 == 0 { + do { + try self.wireUpFramebuffer() + } catch { + // Swallow — we'll try again on the next tick. } } } - timer.resume() - self.idleTimer = timer } // MARK: - Frame capture - private func captureFrame() { + private func captureFrame(force: Bool = false) { guard let desc = pickBestDescriptor() else { return } let surfSel = NSSelectorFromString("framebufferSurface") @@ -230,14 +219,11 @@ final class FrameCapture { // don't spend cycles re-encoding the same pixels back-to-back from the // frame-callback path. BUT: we must still re-emit at the idle floor // (~5 fps) so that downstream consumers keep seeing a live stream — - // see the `idleIntervalMs` doc-comment for why that matters. + // see the `idleInterval` doc-comment for why that matters. let key = ObjectIdentifier(desc) let seed = IOSurfaceGetSeed(surface) - let nowMs = DispatchTime.now().uptimeNanoseconds / 1_000_000 - let sinceLastMs = nowMs &- lastCaptureTimeMs let seedChanged = lastSeeds[key] != seed - let idleRefreshDue = frameCount > 0 && sinceLastMs >= Self.idleIntervalMs - if frameCount > 0, !seedChanged, !idleRefreshDue { return } + if frameCount > 0, !seedChanged, !force { return } lastSeeds[key] = seed let w = IOSurfaceGetWidth(surface) @@ -258,10 +244,11 @@ final class FrameCapture { ) guard status == kCVReturnSuccess, let pb = pixelBuffer?.takeRetainedValue() else { return } - lastCaptureTimeMs = nowMs + lastCaptureTime = .now frameCount += 1 let timestamp = CMTime(value: CMTimeValue(frameCount), timescale: 60) - onFrame?(pb, timestamp) + guard let copy = photocopier.copy(pb) else { return } + onFrame?(copy, timestamp) } func getScreenSize() -> (width: Int, height: Int)? { @@ -308,3 +295,14 @@ final class FrameCapture { }) } } + +@objc protocol FramebufferDescriptor { + @objc(registerScreenCallbacksWithUUID:callbackQueue:frameCallback:surfacesChangedCallback:propertiesChangedCallback:) + func registerScreenCallbacks( + uuid: UUID, + callbackQueue: DispatchQueue, + frameCallback: @convention(block) @escaping () -> Void, + surfacesChangedCallback: @convention(block) @escaping () -> Void, + propertiesChangedCallback: @convention(block) @escaping () -> Void + ) +} diff --git a/packages/serve-sim/Sources/SimNative/H264Encoder.swift b/packages/serve-sim/Sources/SimNative/H264Encoder.swift index 4249da989..aa4513220 100644 --- a/packages/serve-sim/Sources/SimNative/H264Encoder.swift +++ b/packages/serve-sim/Sources/SimNative/H264Encoder.swift @@ -11,7 +11,10 @@ import VideoToolbox /// The incoming buffer wraps SimulatorKit's live framebuffer IOSurface, which /// SimulatorKit recycles in place — VT encodes asynchronously, so we deep-copy /// into a private pooled buffer before submitting to avoid a torn frame race. -final class H264Encoder { +actor H264Encoder { + let queue = DispatchSerialQueue(label: "h264-encoder", qos: .userInteractive) + nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() } + struct Encoded { /// avcC parameter-set blob — emitted once on the first IDR per session. let description: Data? @@ -21,16 +24,11 @@ final class H264Encoder { enum Kind { case keyframe, delta } } - var onEncoded: ((Encoded) -> Void)? - - private let lock = NSLock() private var session: VTCompressionSession? - private var pool: CVPixelBufferPool? private var width: Int32 = 0 private var height: Int32 = 0 private let fps: Int32 private var bitrate: Int - private let stateQueue = DispatchQueue(label: "H264Encoder.state") private var emittedDescription = false private var frameCount: Int64 = 0 @@ -44,8 +42,7 @@ final class H264Encoder { } /// Submit a frame. Returns immediately; `onEncoded` fires on VT's queue. - func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false, completion: (() -> Void)? = nil) { - lock.lock() + func encode(_ source: CVPixelBuffer, forceKeyframe: Bool = false) async throws -> Encoded { let w = Int32(CVPixelBufferGetWidth(source)) let h = Int32(CVPixelBufferGetHeight(source)) if session == nil || w != width || h != height { @@ -53,10 +50,8 @@ final class H264Encoder { height = h rebuildSession() } - guard let session, let copy = copyBuffer(source) else { - lock.unlock() - completion?() - return + guard let session else { + throw Errors.couldNotCreateSession } frameCount += 1 @@ -64,63 +59,39 @@ final class H264Encoder { let frameProps: NSDictionary? = forceKeyframe ? [kVTEncodeFrameOptionKey_ForceKeyFrame: kCFBooleanTrue!] as NSDictionary : nil - lock.unlock() - let status = VTCompressionSessionEncodeFrame( - session, - imageBuffer: copy, - presentationTimeStamp: pts, - duration: .invalid, - frameProperties: frameProps, - infoFlagsOut: nil - ) { [weak self] status, _, sampleBuffer in - defer { completion?() } - guard let self, status == noErr, let sb = sampleBuffer else { return } - if let encoded = self.extract(from: sb) { self.onEncoded?(encoded) } - } - if status != noErr { - completion?() + let buffer: CMSampleBuffer? = await withCheckedContinuation { continuation in + let status = VTCompressionSessionEncodeFrame( + session, + imageBuffer: source, + presentationTimeStamp: pts, + duration: .invalid, + frameProperties: frameProps, + infoFlagsOut: nil + ) { @Sendable status, _, sampleBuffer in + guard status == noErr, let sb = sampleBuffer else { + continuation.resume(returning: nil) + return + } + continuation.resume(returning: sb) + } + if status != noErr { + continuation.resume(returning: nil) + } } + guard let buffer else { throw Errors.encodingFailed } + return try extract(from: buffer) } func stop() { - lock.lock() - defer { lock.unlock() } if let session { VTCompressionSessionInvalidate(session) self.session = nil } - pool = nil } // MARK: - private - /// Deep-copy `source` (which wraps the recycled framebuffer IOSurface) - /// into a private pooled buffer that VT can hold past this call. - private func copyBuffer(_ source: CVPixelBuffer) -> CVPixelBuffer? { - guard let pool else { return nil } - var out: CVPixelBuffer? - guard CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &out) == kCVReturnSuccess, - let dst = out else { return nil } - - CVPixelBufferLockBaseAddress(source, .readOnly) - CVPixelBufferLockBaseAddress(dst, []) - defer { - CVPixelBufferUnlockBaseAddress(dst, []) - CVPixelBufferUnlockBaseAddress(source, .readOnly) - } - guard let src = CVPixelBufferGetBaseAddress(source), - let dstAddr = CVPixelBufferGetBaseAddress(dst) else { return nil } - let srcStride = CVPixelBufferGetBytesPerRow(source) - let dstStride = CVPixelBufferGetBytesPerRow(dst) - let rows = CVPixelBufferGetHeight(source) - let copyBytes = min(srcStride, dstStride) - for row in 0.. Encoded? { + private func extract(from sample: CMSampleBuffer) throws -> Encoded { let isKeyframe = !notSync(sample) - guard let dataBuf = CMSampleBufferGetDataBuffer(sample) else { return nil } + guard let dataBuf = CMSampleBufferGetDataBuffer(sample) else { + throw Errors.invalidSampleBuffer + } var totalLength = 0 var dataPointer: UnsafeMutablePointer? guard CMBlockBufferGetDataPointer( dataBuf, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: &totalLength, dataPointerOut: &dataPointer - ) == noErr, let dataPointer else { return nil } + ) == noErr, let dataPointer else { + throw Errors.invalidSampleBuffer + } let avcc = Data(bytes: dataPointer, count: totalLength) var description: Data? if isKeyframe, let format = CMSampleBufferGetFormatDescription(sample) { let nextDescription = avcCBlob(from: format) - let shouldEmit = stateQueue.sync { () -> Bool in - if emittedDescription { return false } - emittedDescription = nextDescription != nil - return nextDescription != nil - } - if shouldEmit { + if !emittedDescription && nextDescription != nil { + emittedDescription = true description = nextDescription } } @@ -258,4 +216,10 @@ final class H264Encoder { blob.append(contentsOf: pps) return blob } + + enum Errors: Error { + case couldNotCreateSession + case encodingFailed + case invalidSampleBuffer + } } diff --git a/packages/serve-sim/Sources/SimNative/HIDInjector.swift b/packages/serve-sim/Sources/SimNative/HIDInjector.swift index 0a328a779..92ee6d544 100644 --- a/packages/serve-sim/Sources/SimNative/HIDInjector.swift +++ b/packages/serve-sim/Sources/SimNative/HIDInjector.swift @@ -27,7 +27,10 @@ private func hidLog(_ message: @autoclosure () -> String) { /// Apple's Simulator.app always passes NSSize(1.0, 1.0), making ratio = point / 1.0 = point. /// The edge parameter (x4) controls whether iOS treats the touch as a system edge gesture /// (e.g. bottom edge = swipe-to-home on Face ID devices). -final class HIDInjector { +actor HIDInjector { + let queue = DispatchSerialQueue(label: "hid-injector", qos: .userInteractive) + nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() } + private var hidClient: NSObject? private var sendSel: Selector? private var simDevice: NSObject? @@ -146,13 +149,6 @@ final class HIDInjector { static let edgeLeft: UInt32 = 1 // Left edge static let edgeRight: UInt32 = 4 // Right edge - // All HID sends funnel through this one serial queue so concurrent input - // gestures (a scroll drag, a user touch, a button press) can never interleave - // their messages to the shared `hidClient`. One-shot events dispatch a single - // `rawSend`; multi-step gestures (scroll, swipe-home, multi-press buttons) run - // their whole sequence in one block using the synchronous helpers below. - private let inputQueue = DispatchQueue(label: "hid-input") - /// Synchronously hand an already-built Indigo message to the guest, freeing it. /// Must run on `inputQueue`. private func rawSend(_ msg: UnsafeMutableRawPointer) { @@ -188,7 +184,7 @@ final class HIDInjector { func sendTouch(type: String, x: Double, y: Double, screenWidth: Int, screenHeight: Int, edge: UInt32 = 0) { guard let msg = touchMessage(type: type, x: x, y: y, edge: edge) else { return } hidLog("[hid] Sending \(type) at (\(String(format:"%.3f",x)),\(String(format:"%.3f",y)))\(edge > 0 ? " edge=\(edge)" : "")") - inputQueue.async { [self] in rawSend(msg) } + rawSend(msg) } func sendMultiTouch(type: String, x1: Double, y1: Double, x2: Double, y2: Double, screenWidth: Int, screenHeight: Int) { @@ -210,7 +206,7 @@ final class HIDInjector { } hidLog("[hid] Multi-touch \(type) f1=(\(String(format:"%.3f",x1)),\(String(format:"%.3f",y1))) f2=(\(String(format:"%.3f",x2)),\(String(format:"%.3f",y2)))") - inputQueue.async { [self] in rawSend(rawMsg) } + rawSend(rawMsg) } // MARK: - Button events @@ -270,7 +266,7 @@ final class HIDInjector { } hidLog("[hid] Key \(type) usage=0x\(String(usage, radix: 16))") - inputQueue.async { [self] in rawSend(msg) } + rawSend(msg) } // MARK: - Digital Crown events @@ -290,7 +286,7 @@ final class HIDInjector { } hidLog("[hid] Digital Crown delta=\(String(format:"%.4f", delta))") - inputQueue.async { [self] in rawSend(msg) } + rawSend(msg) } // MARK: - Scroll events @@ -339,7 +335,7 @@ final class HIDInjector { /// - dy: Vertical scroll delta in device pixels (positive = content down). /// - anchorX/anchorY: Normalized (0–1) cursor position to begin the drag /// under, so iOS pans the view beneath the pointer. Nil = screen center. - func sendScroll(dx: Double, dy: Double, anchorX: Double?, anchorY: Double?, screenWidth: Int, screenHeight: Int) { + func sendScroll(dx: Double, dy: Double, anchorX: Double?, anchorY: Double?, screenWidth: Int, screenHeight: Int) async { guard dx.isFinite, dy.isFinite, (dx != 0 || dy != 0), screenWidth > 0, screenHeight > 0 else { return } // Finger moves opposite to content: scrolling content down = swipe up. @@ -348,47 +344,47 @@ final class HIDInjector { let aX = clampFinger(anchorX.flatMap { $0.isFinite ? $0 : nil } ?? 0.5) let aY = clampFinger(anchorY.flatMap { $0.isFinite ? $0 : nil } ?? 0.5) - inputQueue.async { [self] in - if !scrollDragActive { - // Anchor a fresh gesture under the cursor so iOS hit-tests the - // right scroll view (e.g. a bottom sheet vs. the map behind it). - scrollAnchorX = aX - scrollAnchorY = aY - scrollFingerX = aX - scrollFingerY = aY - beginDrag(x: scrollFingerX, y: scrollFingerY) - scrollDragActive = true - } - - var nextX = scrollFingerX + stepX - var nextY = scrollFingerY + stepY - - // Near an edge: lift, re-anchor back under the cursor, and continue. - // Re-beginning at the anchor keeps the gesture hit-testing the same view. - if nextX <= HIDInjector.scrollEdgeMargin || nextX >= 1 - HIDInjector.scrollEdgeMargin || - nextY <= HIDInjector.scrollEdgeMargin || nextY >= 1 - HIDInjector.scrollEdgeMargin { - rawSendTouch(type: "end", x: scrollFingerX, y: scrollFingerY) - scrollFingerX = scrollAnchorX - scrollFingerY = scrollAnchorY - beginDrag(x: scrollFingerX, y: scrollFingerY) - nextX = scrollFingerX + stepX - nextY = scrollFingerY + stepY - } - - scrollFingerX = clampFinger(nextX) - scrollFingerY = clampFinger(nextY) - rawSendTouch(type: "move", x: scrollFingerX, y: scrollFingerY) - - // End the drag shortly after the wheel goes idle. - scrollEndWork?.cancel() - let work = DispatchWorkItem { [self] in - guard scrollDragActive else { return } - rawSendTouch(type: "end", x: scrollFingerX, y: scrollFingerY) - scrollDragActive = false - } - scrollEndWork = work - inputQueue.asyncAfter(deadline: .now() + HIDInjector.scrollGestureIdle, execute: work) - } + if !scrollDragActive { + // Anchor a fresh gesture under the cursor so iOS hit-tests the + // right scroll view (e.g. a bottom sheet vs. the map behind it). + scrollAnchorX = aX + scrollAnchorY = aY + scrollFingerX = aX + scrollFingerY = aY + beginDrag(x: scrollFingerX, y: scrollFingerY) + scrollDragActive = true + } + + var nextX = scrollFingerX + stepX + var nextY = scrollFingerY + stepY + + // Near an edge: lift, re-anchor back under the cursor, and continue. + // Re-beginning at the anchor keeps the gesture hit-testing the same view. + if nextX <= HIDInjector.scrollEdgeMargin || nextX >= 1 - HIDInjector.scrollEdgeMargin || + nextY <= HIDInjector.scrollEdgeMargin || nextY >= 1 - HIDInjector.scrollEdgeMargin { + rawSendTouch(type: "end", x: scrollFingerX, y: scrollFingerY) + scrollFingerX = scrollAnchorX + scrollFingerY = scrollAnchorY + beginDrag(x: scrollFingerX, y: scrollFingerY) + nextX = scrollFingerX + stepX + nextY = scrollFingerY + stepY + } + + scrollFingerX = clampFinger(nextX) + scrollFingerY = clampFinger(nextY) + rawSendTouch(type: "move", x: scrollFingerX, y: scrollFingerY) + + // End the drag shortly after the wheel goes idle. + scrollEndWork?.cancel() + let work = DispatchWorkItem { [self] in + guard scrollDragActive else { return } + rawSendTouch(type: "end", x: scrollFingerX, y: scrollFingerY) + scrollDragActive = false + } + scrollEndWork = work + + try? await Task.sleep(for: .seconds(HIDInjector.scrollGestureIdle)) + work.perform() } /// Press an arbitrary hardware button identified by its HID (page, usage), @@ -396,7 +392,7 @@ final class HIDInjector { /// volume up/down, the action button, and the watch side button. /// - phase: "down" / "up" hold the button for natural long-presses (power /// off slider, side-button menus); "press" sends a momentary down+up. - func sendButtonHID(page: UInt32, usage: UInt32, phase: String) { + func sendButtonHID(page: UInt32, usage: UInt32, phase: String) async { guard let arb = hidArbitraryFunc else { print("[hid] Arbitrary HID injection unavailable (page=\(page) usage=\(usage))") return @@ -410,19 +406,17 @@ final class HIDInjector { rawSend(msg) } hidLog("[hid] HID button page=\(page) usage=\(usage) phase=\(phase)") - inputQueue.async { - switch phase { - case "down": emit(1) - case "up": emit(2) - default: - emit(1) - Thread.sleep(forTimeInterval: 0.05) - emit(2) - } + switch phase { + case "down": emit(1) + case "up": emit(2) + default: + emit(1) + try? await Task.sleep(for: .seconds(0.05)) + emit(2) } } - func sendButton(button: String, deviceUDID: String) { + func sendButton(button: String, deviceUDID: String) async { hidLog("[hid] Sending button: \(button)") switch button { @@ -437,44 +431,34 @@ final class HIDInjector { launchSpringBoard(deviceUDID: deviceUDID) case "swipe_home": - inputQueue.async { [self] in - sendSwipeHome() - } + sendSwipeHome() case "app_switcher": if buttonFunc != nil { // Double home press with delay for app switcher - inputQueue.async { [self] in - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) - Thread.sleep(forTimeInterval: 0.15) - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) - } + sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) + try? await Task.sleep(for: .seconds(0.15)) + sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceHome, direction: Self.buttonUp) } else { print("[hid] App switcher not available (IndigoHIDMessageForButton not loaded)") } case "lock": - inputQueue.async { [self] in - sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonUp) - } + sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceLock, direction: Self.buttonUp) case "siri": // Holding Siri for ~300ms matches Simulator.app's "hold side button // to invoke Siri" gesture; a tap is ignored. - inputQueue.async { [self] in - sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) - Thread.sleep(forTimeInterval: 0.3) - sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) - } + sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonDown) + try? await Task.sleep(for: .seconds(0.3)) + sendHIDButton(eventSource: Self.buttonSourceSiri, direction: Self.buttonUp) case "side_button": - inputQueue.async { [self] in - sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) - } + sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceSideButton, direction: Self.buttonUp) default: print("[hid] Unknown button: \(button)") @@ -515,10 +499,8 @@ final class HIDInjector { print("[hid] Software keyboard toggle unavailable (IndigoHIDMessageForButton not loaded)") return } - inputQueue.async { [self] in - sendHIDButton(eventSource: Self.buttonSourceSoftwareKeyboard, direction: Self.buttonDown) - sendHIDButton(eventSource: Self.buttonSourceSoftwareKeyboard, direction: Self.buttonUp) - } + sendHIDButton(eventSource: Self.buttonSourceSoftwareKeyboard, direction: Self.buttonDown) + sendHIDButton(eventSource: Self.buttonSourceSoftwareKeyboard, direction: Self.buttonUp) } /// Ask CoreSimulator to broadcast a memory warning to the simulated OS. diff --git a/packages/serve-sim/Sources/SimNative/PixelBufferUtils.swift b/packages/serve-sim/Sources/SimNative/PixelBufferUtils.swift new file mode 100644 index 000000000..fa1c66941 --- /dev/null +++ b/packages/serve-sim/Sources/SimNative/PixelBufferUtils.swift @@ -0,0 +1,62 @@ +import CoreVideo + +struct Dimensions: Hashable, Sendable { + var width: Int + var height: Int +} + +extension CVPixelBuffer { + var dimensions: Dimensions { + Dimensions(width: CVPixelBufferGetWidth(self), height: CVPixelBufferGetHeight(self)) + } +} + +struct Photocopier { + private var _pool: CVPixelBufferPool? + private var dimensions: Dimensions? + + init() {} + + private mutating func pool(dimensions: Dimensions) -> CVPixelBufferPool? { + if let _pool, self.dimensions == dimensions { + return _pool + } + let attrs: [String: Any] = [ + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, + kCVPixelBufferWidthKey as String: Int(dimensions.width), + kCVPixelBufferHeightKey as String: Int(dimensions.height), + kCVPixelBufferIOSurfacePropertiesKey as String: [:], + ] + var newPool: CVPixelBufferPool? + CVPixelBufferPoolCreate(kCFAllocatorDefault, nil, attrs as CFDictionary, &newPool) + self._pool = newPool + self.dimensions = dimensions + return newPool + } + + /// Deep-copy `source` (which wraps the recycled framebuffer IOSurface) + /// into a private pooled buffer that sinks can retain. + mutating func copy(_ source: CVPixelBuffer) -> CVPixelBuffer? { + guard let pool = self.pool(dimensions: source.dimensions) else { return nil } + var out: CVPixelBuffer? + guard CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &out) == kCVReturnSuccess, + let dst = out else { return nil } + + CVPixelBufferLockBaseAddress(source, .readOnly) + CVPixelBufferLockBaseAddress(dst, []) + defer { + CVPixelBufferUnlockBaseAddress(dst, []) + CVPixelBufferUnlockBaseAddress(source, .readOnly) + } + guard let srcAddr = CVPixelBufferGetBaseAddress(source), + let dstAddr = CVPixelBufferGetBaseAddress(dst) else { return nil } + let srcStride = CVPixelBufferGetBytesPerRow(source) + let dstStride = CVPixelBufferGetBytesPerRow(dst) + let rows = CVPixelBufferGetHeight(source) + let copyBytes = min(srcStride, dstStride) + for row in 0.. Void)? +actor VideoEncoder { + // makes sure we don't block the main thread / cooperative thread pool + let queue = DispatchSerialQueue(label: "video-encoder", qos: .userInteractive) + + nonisolated var unownedExecutor: UnownedSerialExecutor { queue.asUnownedSerialExecutor() } + private let quality: CGFloat init(quality: CGFloat = 0.7) { self.quality = quality } - func setup(width: Int32, height: Int32, fps: Int, - onEncodedFrame: @escaping (Data) -> Void) { - self.onEncodedFrame = onEncodedFrame - print("[encoder] JPEG encoder ready at \(width)x\(height) (quality: \(quality))") - } - - func encode(pixelBuffer: CVPixelBuffer) { + func encode(pixelBuffer: CVPixelBuffer) throws -> Data { CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly) defer { CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly) } let width = CVPixelBufferGetWidth(pixelBuffer) let height = CVPixelBufferGetHeight(pixelBuffer) - guard let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) else { return } + guard let baseAddress = CVPixelBufferGetBaseAddress(pixelBuffer) else { + throw Errors.invalidPixelBuffer + } let bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer) let colorSpace = CGColorSpaceCreateDeviceRGB() @@ -37,17 +37,24 @@ final class VideoEncoder { bytesPerRow: bytesPerRow, space: colorSpace, bitmapInfo: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue - ), let cgImage = context.makeImage() else { return } + ), let cgImage = context.makeImage() else { + throw Errors.encodingFailed + } let data = NSMutableData() - guard let dest = CGImageDestinationCreateWithData(data as CFMutableData, "public.jpeg" as CFString, 1, nil) else { return } + guard let dest = CGImageDestinationCreateWithData(data as CFMutableData, "public.jpeg" as CFString, 1, nil) else { + throw Errors.encodingFailed + } CGImageDestinationAddImage(dest, cgImage, [kCGImageDestinationLossyCompressionQuality: quality] as CFDictionary) - guard CGImageDestinationFinalize(dest) else { return } + guard CGImageDestinationFinalize(dest) else { + throw Errors.encodingFailed + } - onEncodedFrame?(data as Data) + return data as Data } - func stop() { - onEncodedFrame = nil + enum Errors: Error { + case invalidPixelBuffer + case encodingFailed } } diff --git a/packages/serve-sim/Sources/SimNative/sim-capture.swift b/packages/serve-sim/Sources/SimNative/sim-capture.swift deleted file mode 100644 index cff48ac94..000000000 --- a/packages/serve-sim/Sources/SimNative/sim-capture.swift +++ /dev/null @@ -1,224 +0,0 @@ -import Foundation -import CoreVideo -import CoreMedia - -// The capture + encode engine, reused verbatim from SimStreamHelper. Replicates -// main.swift's frameHandler: MJPEG always encodes while clients exist; H.264 runs -// only while AVCC is active. Encoded bytes (JPEG, or natively-framed AVCC -// envelopes) are handed back through a Swift closure on a native encode thread; -// the node-swift binding (sim-module.swift) marshals them onto the JS thread via -// a NodeAsyncQueue (threadsafe function). - -/// (codec, data, width, height, flags) -> Void, invoked on a native encode -/// thread. codec: 0 = MJPEG, 1 = AVCC. flags (AVCC): bit0 = description, -/// bit1 = keyframe. `data` is a freshly-copied value safe to retain. -typealias SimFrameCallback = (Int32, Data, Int32, Int32, Int32) -> Void - -final class CaptureEngine { - static let codecMJPEG: Int32 = 0 - static let codecAVCC: Int32 = 1 - static let flagDescription: Int32 = 1 << 0 - static let flagKeyframe: Int32 = 1 << 1 - - private let deviceUDID: String - private let onFrame: SimFrameCallback - - private let frameCapture = FrameCapture() - private let videoEncoder = VideoEncoder(quality: 0.7) - private let h264Encoder = H264Encoder(fps: 60) - private let encodeQueue = DispatchQueue(label: "napi.encode", qos: .userInteractive) - private let h264Queue = DispatchQueue(label: "napi.encode.h264", qos: .userInteractive) - private static let h264EncodeTimeoutMs = 500 - - // Mirrors main.swift's globals; mutated from the capture queue, read from the - // encode queues. Benign races (same pattern as the standalone helper). - private var screenWidth = 0 - private var screenHeight = 0 - private var encoderReady = false - private var encoding = false // MJPEG backpressure - private var h264Encoding = false // H.264 backpressure - private var forceKeyframe = false - private var avccActive = false - private var h264FrameToken: UInt64 = 0 - private var started = false - private var stopped = false - - init(deviceUDID: String, onFrame: @escaping SimFrameCallback) { - self.deviceUDID = deviceUDID - self.onFrame = onFrame - - h264Encoder.onEncoded = { [weak self] encoded in - guard let self else { return } - if let description = encoded.description { - self.emit(codec: Self.codecAVCC, - data: AVCCEnvelope.description(avcc: description), - flags: Self.flagDescription) - } - switch encoded.kind { - case .keyframe: - self.emit(codec: Self.codecAVCC, data: AVCCEnvelope.keyframe(avcc: encoded.avcc), - flags: Self.flagKeyframe) - case .delta: - self.emit(codec: Self.codecAVCC, data: AVCCEnvelope.delta(avcc: encoded.avcc), flags: 0) - } - } - } - - /// Hand encoded bytes to the binding. Gated by `stopped` so no callback fires - /// once teardown has begun. - private func emit(codec: Int32, data: Data, flags: Int32) { - if stopped { return } - onFrame(codec, data, Int32(screenWidth), Int32(screenHeight), flags) - } - - func start() throws { - guard !started else { return } - // Latch `started` only after capture actually begins: if start() throws - // (e.g. device not booted), a later retry should still be allowed. - try frameCapture.start(deviceUDID: deviceUDID) { [weak self] pixelBuffer, _ in - self?.handleFrame(pixelBuffer) - } - started = true - } - - private func handleFrame(_ pixelBuffer: CVPixelBuffer) { - let w = CVPixelBufferGetWidth(pixelBuffer) - let h = CVPixelBufferGetHeight(pixelBuffer) - - if !encoderReady || w != screenWidth || h != screenHeight { - screenWidth = w - screenHeight = h - videoEncoder.stop() - videoEncoder.setup(width: Int32(w), height: Int32(h), fps: 60) { [weak self] jpeg in - self?.emit(codec: Self.codecMJPEG, data: jpeg, flags: 0) - } - encoderReady = true - } - - let h264Request = reserveH264EncodeIfNeeded() - let shouldEncodeJpeg = encoderReady && !encoding - if !shouldEncodeJpeg && h264Request == nil { return } - - guard let stableFrame = copyPixelBuffer(pixelBuffer) else { - if let h264Request { - finishH264Encode(token: h264Request.token, restoreKeyframe: h264Request.forceKeyframe) - } - return - } - - if shouldEncodeJpeg { - encoding = true - encodeQueue.async { [weak self] in - guard let self else { return } - self.videoEncoder.encode(pixelBuffer: stableFrame) - self.encoding = false - } - } - - // H.264 runs only while a viewer wants AVCC, so an all-MJPEG session pays - // no VideoToolbox cost. - if let h264Request { - h264Queue.async { [weak self] in - guard let self else { return } - self.h264Encoder.encode(stableFrame, forceKeyframe: h264Request.forceKeyframe) { - self.finishH264Encode(token: h264Request.token) - } - self.scheduleH264EncodeTimeout(token: h264Request.token) - } - } - } - - /// Copy the live Simulator IOSurface immediately on the capture queue. The - /// encoders run later and SimulatorKit recycles/mutates that IOSurface in - /// place, so passing the wrapper CVPixelBuffer across queues can encode a - /// half-updated frame. - private func copyPixelBuffer(_ source: CVPixelBuffer) -> CVPixelBuffer? { - let width = CVPixelBufferGetWidth(source) - let height = CVPixelBufferGetHeight(source) - let pixelFormat = CVPixelBufferGetPixelFormatType(source) - let attrs: [String: Any] = [ - kCVPixelBufferPixelFormatTypeKey as String: pixelFormat, - kCVPixelBufferWidthKey as String: width, - kCVPixelBufferHeightKey as String: height, - kCVPixelBufferCGImageCompatibilityKey as String: true, - kCVPixelBufferCGBitmapContextCompatibilityKey as String: true, - ] - var out: CVPixelBuffer? - guard CVPixelBufferCreate( - kCFAllocatorDefault, width, height, pixelFormat, attrs as CFDictionary, &out - ) == kCVReturnSuccess, let dst = out else { return nil } - - CVPixelBufferLockBaseAddress(source, .readOnly) - CVPixelBufferLockBaseAddress(dst, []) - defer { - CVPixelBufferUnlockBaseAddress(dst, []) - CVPixelBufferUnlockBaseAddress(source, .readOnly) - } - guard let srcAddr = CVPixelBufferGetBaseAddress(source), - let dstAddr = CVPixelBufferGetBaseAddress(dst) else { return nil } - let srcStride = CVPixelBufferGetBytesPerRow(source) - let dstStride = CVPixelBufferGetBytesPerRow(dst) - let rows = CVPixelBufferGetHeight(source) - let copyBytes = min(srcStride, dstStride) - for row in 0.. (forceKeyframe: Bool, token: UInt64)? { - h264Queue.sync { - guard avccActive, !h264Encoding else { return nil } - h264Encoding = true - h264FrameToken &+= 1 - let token = h264FrameToken - let force = forceKeyframe - forceKeyframe = false - return (forceKeyframe: force, token: token) - } - } - - private func finishH264Encode(token: UInt64, restoreKeyframe: Bool = false) { - h264Queue.async { [weak self] in - guard let self, self.h264FrameToken == token else { return } - self.h264Encoding = false - if restoreKeyframe { self.forceKeyframe = true } - } - } - - private func scheduleH264EncodeTimeout(token: UInt64) { - h264Queue.asyncAfter(deadline: .now().advanced(by: .milliseconds(Self.h264EncodeTimeoutMs))) { [weak self] in - guard let self, self.h264FrameToken == token else { return } - self.h264Encoding = false - } - } - - /// Toggle H.264 encoding. Turning it on forces the next frame to an IDR so a - /// freshly-connected decoder has a keyframe to start from. - func setAvccActive(_ active: Bool) { - h264Queue.async { [weak self] in - guard let self else { return } - if active && !self.avccActive { self.forceKeyframe = true } - self.avccActive = active - } - } - - func requestKeyframe() { - h264Queue.async { [weak self] in self?.forceKeyframe = true } - } - - func screenSize() -> (Int, Int) { (screenWidth, screenHeight) } - - /// Halt frame production and drain the encode queues so no callback can fire - /// after this returns — the N-API layer relies on that before releasing the - /// threadsafe function. - func stop() { - if stopped { return } - stopped = true - frameCapture.stop() - encodeQueue.sync {} - h264Queue.sync {} - videoEncoder.stop() - h264Encoder.stop() - } -} diff --git a/packages/serve-sim/Sources/SimNative/sim-module.swift b/packages/serve-sim/Sources/SimNative/sim-module.swift index 9559f9e6b..32f4d1190 100644 --- a/packages/serve-sim/Sources/SimNative/sim-module.swift +++ b/packages/serve-sim/Sources/SimNative/sim-module.swift @@ -27,60 +27,60 @@ private func u32(_ v: Int) -> UInt32 { @NodeConstructor init(_ udid: String) throws { self.udid = udid injector = HIDInjector() - try injector.setup(deviceUDID: udid) + Task { try await injector.setup(deviceUDID: udid) } } @NodeMethod func touch(_ type: String, _ x: Double, _ y: Double, - _ w: Int, _ h: Int, _ edge: Int) { - injector.sendTouch(type: type, x: x, y: y, + _ w: Int, _ h: Int, _ edge: Int) async { + await injector.sendTouch(type: type, x: x, y: y, screenWidth: w, screenHeight: h, edge: u32(edge)) } @NodeMethod func multiTouch(_ type: String, _ x1: Double, _ y1: Double, - _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) { - injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2, + _ x2: Double, _ y2: Double, _ w: Int, _ h: Int) async { + await injector.sendMultiTouch(type: type, x1: x1, y1: y1, x2: x2, y2: y2, screenWidth: w, screenHeight: h) } - @NodeMethod func button(_ button: String) { - injector.sendButton(button: button, deviceUDID: udid) + @NodeMethod func button(_ button: String) async { + await injector.sendButton(button: button, deviceUDID: udid) } - @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) { - injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase) + @NodeMethod func buttonHid(_ page: Int, _ usage: Int, _ phase: String) async { + await injector.sendButtonHID(page: u32(page), usage: u32(usage), phase: phase) } - @NodeMethod func key(_ type: String, _ usage: Int) { - injector.sendKey(type: type, usage: u32(usage)) + @NodeMethod func key(_ type: String, _ usage: Int) async { + await injector.sendKey(type: type, usage: u32(usage)) } /// NaN anchorX/anchorY mean "center" (the Swift API's nil). @NodeMethod func scroll(_ dx: Double, _ dy: Double, - _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) { - injector.sendScroll(dx: dx, dy: dy, + _ anchorX: Double, _ anchorY: Double, _ w: Int, _ h: Int) async { + await injector.sendScroll(dx: dx, dy: dy, anchorX: anchorX.isNaN ? nil : anchorX, anchorY: anchorY.isNaN ? nil : anchorY, screenWidth: w, screenHeight: h) } - @NodeMethod func digitalCrown(_ delta: Double) { - injector.sendDigitalCrown(delta: delta) + @NodeMethod func digitalCrown(_ delta: Double) async { + await injector.sendDigitalCrown(delta: delta) } - @NodeMethod func orientation(_ orientation: Int) -> Bool { - injector.sendOrientation(orientation: u32(orientation)) + @NodeMethod func orientation(_ orientation: Int) async -> Bool { + await injector.sendOrientation(orientation: u32(orientation)) } - @NodeMethod func memoryWarning() { - injector.simulateMemoryWarning() + @NodeMethod func memoryWarning() async { + await injector.simulateMemoryWarning() } - @NodeMethod func softwareKeyboard() { - injector.toggleSoftwareKeyboard() + @NodeMethod func softwareKeyboard() async { + await injector.toggleSoftwareKeyboard() } - @NodeMethod func caDebug(_ name: String, _ enabled: Bool) -> Bool { - injector.setCADebugOption(name: name, enabled: enabled) + @NodeMethod func caDebug(_ name: String, _ enabled: Bool) async -> Bool { + await injector.setCADebugOption(name: name, enabled: enabled) } } @@ -93,64 +93,94 @@ private func u32(_ v: Int) -> UInt32 { /// (codec, Buffer, width, height, flags). @NodeClass @NodeActor final class SimCapture { private let engine: CaptureEngine - private let onFrame: NodeFunction private let queue: NodeAsyncQueue - @NodeConstructor init(_ udid: String, _ onFrame: NodeFunction) throws { + @NodeConstructor init(_ udid: String) throws { // unref'd by NodeAsyncQueue's init, so the frame pipeline alone won't // keep the event loop alive. Bounded queue + blocking AVCC preserves // inter-frame ordering; MJPEG is nonblocking and drops under backpressure. let queue = try NodeAsyncQueue(label: "simCapture", maxQueueSize: 16) - self.onFrame = onFrame self.queue = queue - - // Capture the locals (not self) so the closure can be built before the - // engine property is initialized, and so it holds no strong ref to self. - engine = CaptureEngine(deviceUDID: udid) { codec, data, w, h, flags in - // Runs on a native encode thread. AVCC is inter-frame H.264 — dropping - // a delta corrupts the decoder until the next IDR — so deliver it - // blocking; MJPEG is stateless and safe to drop. We copy the bytes - // into a managed Buffer (NodeBuffer(copying:)) on the JS thread: - // external buffers crash Bun's GC under frame churn, and the - // production CLI is a bun-compiled binary. - let blocking = codec == CaptureEngine.codecAVCC - try? queue.run(blocking: blocking) { - _ = try? onFrame.call([ - Int(codec), try NodeBuffer(copying: data), - Int(w), Int(h), Int(flags), - ]) + self.engine = CaptureEngine(deviceUDID: udid) + } + + // returns a function that can be called to unsubscribe + @NodeMethod func subscribe( + codec: Int, + onFrame: NodeFunction + ) async throws -> NodeFunction { + let codecMJPEG: Int = 0 + let codecAVCC: Int = 1 + + var buffer = try CaptureBuffer(initialCapacity: 1024 * 1024) + let unsubscribe: @Sendable () async -> Void + switch codec { + case codecMJPEG: + unsubscribe = await engine.addMJPEGConsumer { [self] dimensions, data in + try? await queue.run { + let array = try buffer.setData(data) + _ = try? await onFrame.call([ + array, + Int(dimensions.width), Int(dimensions.height), + 0, + ]).as(NodePromise.self)?.value + } + } + case codecAVCC: + unsubscribe = await engine.addAVCCConsumer { [self] dimensions, data, flags in + try? await queue.run { + let array = try buffer.setData(data) + _ = try? await onFrame.call([ + array, + Int(dimensions.width), Int(dimensions.height), + Int(flags), + ]).as(NodePromise.self)?.value + } } + default: + throw Errors.invalidCodec } + return try NodeFunction { await unsubscribe() } } - @NodeMethod func start() throws { - try engine.start() + @NodeMethod func start() async throws { + try await engine.start() } - @NodeMethod func setAvccActive(_ active: Bool) { - engine.setAvccActive(active) + @NodeMethod func stop() async { + await engine.stop() } - @NodeMethod func requestKeyframe() { - engine.requestKeyframe() + deinit { + Task { [engine] in await engine.stop() } } - @NodeMethod func screenSize() -> [String: any NodePropertyConvertible] { - let (w, h) = engine.screenSize() - return ["width": w, "height": h] + enum Errors: Error { + case invalidCodec } +} + +@NodeActor private struct CaptureBuffer { + private var buffer: NodeArrayBuffer - @NodeMethod func stop() { - engine.stop() + init(initialCapacity: Int) throws { + buffer = try NodeArrayBuffer(capacity: initialCapacity) } - deinit { - // Abort the queue first so any encode thread blocked in `run` unblocks - // (its call returns .closing and the frame is dropped); then drain the - // encoders so nothing can fire afterwards. The tsfn is released when - // `queue` deinitializes after this body. - try? queue.close() - engine.stop() + mutating func setData(_ data: Data) throws -> NodeTypedArray { + let hadSpace = try buffer.withUnsafeMutableBytes { buffer in + guard data.count <= buffer.count else { return false } + _ = data.copyBytes(to: buffer) + return true + } + + if !hadSpace { + // allocate a new buffer with sufficient capacity. old buffer will be GC'd. + buffer = try NodeArrayBuffer(capacity: data.count) + _ = try buffer.withUnsafeMutableBytes { data.copyBytes(to: $0) } + } + + return try NodeTypedArray(for: buffer, count: data.count) } } diff --git a/packages/serve-sim/build.ts b/packages/serve-sim/build.ts index 0b1e41b05..7d4cd4649 100644 --- a/packages/serve-sim/build.ts +++ b/packages/serve-sim/build.ts @@ -129,6 +129,7 @@ const mwResult = await Bun.build({ outdir: distDir, external: ["fs", "path", "os", "child_process", "url", "net", "tls", "crypto", "stream", "events", "http", "https", "zlib", "buffer", "module", "ws"], define: PREVIEW_DEFINE, + sourcemap: "linked", }); if (!mwResult.success) { console.error("Middleware build failed:"); @@ -155,6 +156,7 @@ const binJsResult = await Bun.build({ naming: "serve-sim.js", external: ["fs", "path", "os", "child_process", "url", "net", "tls", "crypto", "stream", "events", "http", "https", "zlib", "buffer", "module", "ws"], define: PREVIEW_DEFINE, + sourcemap: "linked", }); if (!binJsResult.success) { console.error("Bin JS build failed:"); diff --git a/packages/serve-sim/dev.ts b/packages/serve-sim/dev.ts index 91c761293..de8f12e06 100644 --- a/packages/serve-sim/dev.ts +++ b/packages/serve-sim/dev.ts @@ -201,8 +201,8 @@ function scheduleWatchedBuild() { // ─── HTML shell ─── -function buildHtml(selectedDevice?: string | null): string { - const state = selectServeSimState(readServeSimStates(), selectedDevice); +async function buildHtml(selectedDevice?: string | null): Promise { + const state = selectServeSimState(await readServeSimStates(), selectedDevice); // Even with no helper attached the page polls the host (list/boot devices) // and streams `/api/events` over the control socket, so it always needs the // basePath + exec token. @@ -251,14 +251,14 @@ function handleDevReload(req: IncomingMessage, res: ServerResponse): void { // Dev-only routes intercept first; everything else falls through to the // production middleware — including `/grid/api/start`, which now boots + serves // the device in-process (no spawned helper), so no dev override is needed. -function devMiddleware(req: IncomingMessage, res: ServerResponse, next: () => void): void { +async function devMiddleware(req: IncomingMessage, res: ServerResponse, next: () => Promise): Promise { const path = (req.url ?? "").split("?")[0]; if (path === "/__dev/reload") return handleDevReload(req, res); if (path === "/" || path === "") { const device = new URLSearchParams((req.url ?? "").split("?")[1] ?? "").get("device"); res.writeHead(200, { "Content-Type": "text/html; charset=utf-8", "Cache-Control": "no-store" }); - res.end(buildHtml(device)); + res.end(await buildHtml(device)); return; } diff --git a/packages/serve-sim/src/__tests__/exec-auth.test.ts b/packages/serve-sim/src/__tests__/exec-auth.test.ts index c77695d3e..ef534b55f 100644 --- a/packages/serve-sim/src/__tests__/exec-auth.test.ts +++ b/packages/serve-sim/src/__tests__/exec-auth.test.ts @@ -7,7 +7,7 @@ async function withServer(fn: (origin: string) => Promise): Promise { const TOKEN = "test-token-abc123"; const handler = simMiddleware({ basePath: "/", execToken: TOKEN }); const server = createServer((req, res) => { - handler(req, res, () => { + handler(req, res, async () => { if (!res.headersSent) res.statusCode = 404; res.end("Not found"); }); diff --git a/packages/serve-sim/src/device-session.ts b/packages/serve-sim/src/device-session.ts index 4530c0d09..c6b2ba3d7 100644 --- a/packages/serve-sim/src/device-session.ts +++ b/packages/serve-sim/src/device-session.ts @@ -15,7 +15,14 @@ * original byte-for-byte so the existing browser client is unchanged. */ import type { IncomingMessage, ServerResponse } from "http"; -import { NativeCapture, NativeHid, Orientation, axDescribeAsync, axFrontmostAsync, type NativeFrame } from "./native"; +import { + NativeCapture, + NativeHid, + Orientation, + axDescribeAsync, + axFrontmostAsync, + type MjpegFrame, +} from "./native"; /** * Minimal WebSocket surface the HID input channel needs. Satisfied by both the @@ -36,10 +43,6 @@ const CORS = { "Access-Control-Allow-Headers": "Content-Type", }; -// Don't let a stalled viewer's socket buffer grow without bound — drop frames -// for a client that's this far behind rather than balloon memory. -const MAX_CLIENT_BACKLOG = 8 * 1024 * 1024; - // AVCC seed tag (StreamFormat.AVCCEnvelope.seedTag). description/keyframe/delta // envelopes are framed natively; only the on-connect JPEG seed is built here. const AVCC_SEED_TAG = 0x04; @@ -53,11 +56,11 @@ function mjpegHeader(jpegLength: number): Buffer { return Buffer.from(`--frame\r\nContent-Type: image/jpeg\r\nContent-Length: ${jpegLength}\r\n\r\n`, "ascii"); } -function avccSeed(jpeg: Buffer): Buffer { +function avccSeed(jpeg: Uint8Array): Buffer { const out = Buffer.allocUnsafe(5 + jpeg.length); out.writeUInt32BE(jpeg.length + 1, 0); // length covers the tag byte + payload out[4] = AVCC_SEED_TAG; - jpeg.copy(out, 5); + out.set(jpeg, 5); return out; } @@ -68,92 +71,97 @@ const ORIENTATION_BY_NAME: Record = { landscape_right: Orientation.landscapeRight, }; +function waitForDrain(res: ServerResponse): Promise { + if (res.writableEnded || res.destroyed || !res.writableNeedDrain) return Promise.resolve(); + + return new Promise((resolve) => { + const done = () => { + cleanup(); + resolve(); + }; + const cleanup = () => { + res.off("drain", done); + res.off("close", done); + res.off("error", done); + }; + res.once("drain", done); + res.once("close", done); + res.once("error", done); + }); +} + export class DeviceSession { private readonly capture: NativeCapture; private readonly hid: NativeHid; - private started = false; + private unsubscribeMjpeg?: () => void; + private phase: "unstarted" | "running" | "stopped" = "unstarted"; private width = 0; private height = 0; private orientation = "portrait"; - private latestJpeg: Buffer | null = null; - private cachedAvccDescription: Buffer | null = null; - private readonly mjpegClients = new Set(); - private readonly avccClients = new Set(); + private latestJpegBuffer: Buffer | null = null; + private latestJpegLength = 0; private readonly hidSockets = new Set(); constructor(public readonly udid: string) { this.hid = new NativeHid(udid); - this.capture = new NativeCapture(udid, (f) => this.onFrame(f)); + this.capture = new NativeCapture(udid); } /** Begin capture. Throws if the device isn't booted. Idempotent. */ start(): void { - if (this.started) return; + if (this.phase !== "unstarted") return; this.capture.start(); - this.started = true; + void (async () => { + const unsubscribe = await this.capture.subscribeMjpeg((frame) => this.onSharedMjpegFrame(frame)); + if (this.phase === "running") { // only if someone hasn't already stopped the capture + this.unsubscribeMjpeg = unsubscribe; + } else { + unsubscribe(); + } + })(); + this.phase = "running"; } close(): void { - for (const res of this.mjpegClients) res.end(); - for (const res of this.avccClients) res.end(); + if (this.phase !== "running") return; for (const ws of this.hidSockets) ws.close(); - this.mjpegClients.clear(); - this.avccClients.clear(); + this.unsubscribeMjpeg?.(); this.hidSockets.clear(); this.capture.stop(); + this.phase = "stopped"; } - // ── Frame fan-out ──────────────────────────────────────────────────────── + // ── Frame handling ─────────────────────────────────────────────────────── - private onFrame(f: NativeFrame): void { - if (f.codec === "mjpeg") { - this.latestJpeg = f.data; - if (f.width !== this.width || f.height !== this.height) { - this.width = f.width; - this.height = f.height; - this.broadcastConfig(); - } - if (this.mjpegClients.size === 0) return; - // Build only the small header once; the JPEG itself is written by - // reference to every client, avoiding a full-frame copy per frame. - const header = mjpegHeader(f.data.length); - for (const res of this.mjpegClients) this.writeMjpegFrame(res, header, f.data); - } else { - if (f.isDescription) this.cachedAvccDescription = f.data; - for (const res of this.avccClients) this.writeAvccFrame(res, f.data); + private async onSharedMjpegFrame(frame: MjpegFrame): Promise { + const { width, height, data: jpeg } = frame; + + if (width !== this.width || height !== this.height) { + this.width = width; + this.height = height; + this.broadcastConfig(); } + + if (!this.latestJpegBuffer || this.latestJpegBuffer.length < jpeg.length) { + const currentCapacity = this.latestJpegBuffer?.length ?? 0; + this.latestJpegBuffer = Buffer.allocUnsafe(Math.max(jpeg.length, currentCapacity * 2)); + } + this.latestJpegBuffer.set(jpeg, 0); + this.latestJpegLength = jpeg.length; + } + + private latestJpeg(): Buffer | null { + if (!this.latestJpegBuffer) return null; + return this.latestJpegBuffer.subarray(0, this.latestJpegLength); } /** Write a multipart JPEG part (header + shared frame + boundary) without copying the JPEG. */ - private writeMjpegFrame(res: ServerResponse, header: Buffer, jpeg: Buffer): void { - if (res.writableEnded || res.writableLength > MAX_CLIENT_BACKLOG) return; - res.cork(); - res.write(header); + private writeMjpegFrame(res: ServerResponse, jpeg: Uint8Array): void { + res.write(mjpegHeader(jpeg.length)); res.write(jpeg); res.write(MJPEG_TRAILER); - res.uncork(); - } - - /** - * Write an AVCC chunk. AVCC is inter-frame H.264, so dropping a chunk corrupts - * the decoder until the next IDR (visible tearing). Rather than drop, evict a - * client whose socket is backed up: it reconnects via handleAvcc and is - * re-seeded with the cached description + a fresh keyframe, yielding a clean - * stream instead of a corrupted one. - */ - private writeAvccFrame(res: ServerResponse, chunk: Buffer): void { - if (res.writableEnded) { - this.avccClients.delete(res); - return; - } - if (res.writableLength > MAX_CLIENT_BACKLOG) { - this.avccClients.delete(res); - res.end(); - return; - } - res.write(chunk); } // ── HTTP handlers ──────────────────────────────────────────────────────── @@ -166,11 +174,18 @@ export class DeviceSession { Connection: "keep-alive", ...CORS, }); - this.mjpegClients.add(res); - if (this.latestJpeg) this.writeMjpegFrame(res, mjpegHeader(this.latestJpeg.length), this.latestJpeg); // paint immediately - const drop = () => this.mjpegClients.delete(res); - res.on("close", drop); - res.on("error", drop); + + void (async () => { + const latestJpeg = this.latestJpeg(); + if (latestJpeg) this.writeMjpegFrame(res, latestJpeg); // paint immediately + const unsubscribe = await this.capture.subscribeMjpeg(async (frame) => { + await waitForDrain(res); + this.writeMjpegFrame(res, frame.data); + }); + if (res.writableEnded) unsubscribe(); + res.on("close", unsubscribe); + res.on("error", unsubscribe); + })(); } handleAvcc(_req: IncomingMessage, res: ServerResponse): void { @@ -180,19 +195,21 @@ export class DeviceSession { Connection: "keep-alive", ...CORS, }); - this.avccClients.add(res); - this.capture.setAvccActive(true); - // Seed with the current screen, replay the cached decoder config, then force - // an IDR so the freshly-configured decoder has a keyframe to start from. - if (this.latestJpeg) res.write(avccSeed(this.latestJpeg)); - if (this.cachedAvccDescription) res.write(this.cachedAvccDescription); - this.capture.requestKeyframe(); - const drop = () => { - this.avccClients.delete(res); - if (this.avccClients.size === 0) this.capture.setAvccActive(false); - }; - res.on("close", drop); - res.on("error", drop); + + void (async () => { + // Seed with the current screen; the per-client native AVCC subscription + // starts with its own decoder config and keyframe. + const latestJpeg = this.latestJpeg(); + if (latestJpeg) res.write(avccSeed(latestJpeg)); + + const unsubscribe = await this.capture.subscribeAvcc(async (frame) => { + await waitForDrain(res); + res.write(frame.data); + }); + if (res.writableEnded) unsubscribe(); + res.on("close", unsubscribe); + res.on("error", unsubscribe); + })(); } handleConfig(_req: IncomingMessage, res: ServerResponse): void { @@ -237,7 +254,7 @@ export class DeviceSession { ws.on("error", () => this.hidSockets.delete(ws)); } - private handleHidMessage(data: Buffer): void { + private async handleHidMessage(data: Buffer): Promise { if (data.length < 1) return; const tag = data[0]; const body = data.length > 1 ? data.subarray(1) : null; @@ -282,7 +299,7 @@ export class DeviceSession { const m = json<{ orientation: string }>(); if (!m) break; const value = ORIENTATION_BY_NAME[m.orientation]; - if (value != null && this.hid.orientation(value)) { + if (value != null && await this.hid.orientation(value)) { if (m.orientation !== this.orientation) { this.orientation = m.orientation; this.broadcastConfig(); diff --git a/packages/serve-sim/src/middleware.ts b/packages/serve-sim/src/middleware.ts index 432d86e77..8044f3abf 100644 --- a/packages/serve-sim/src/middleware.ts +++ b/packages/serve-sim/src/middleware.ts @@ -27,9 +27,9 @@ import { UI_OPTIONS, getUiStatus, normalizeUiValue, setUiOption } from "./ui-set type SimReq = IncomingMessage; type SimRes = ServerResponse; -type SimNext = (err?: unknown) => void; +type SimNext = (err?: unknown) => Promise; export type SimMiddleware = { - (req: SimReq, res: SimRes, next?: SimNext): void; + (req: SimReq, res: SimRes, next?: SimNext): Promise; handleUpgrade(req: SimReq, socket: Socket, head: Buffer): void; }; @@ -220,18 +220,27 @@ export function matchInstalledAppByDisplayName( // The middleware runs inside the user's dev server (Metro etc.) and // readServeSimStates() is called on every /api and every page load. let bootedSnapshot: { at: number; booted: Set | null } = { at: 0, booted: null }; -function getBootedUdids(): Set | null { +async function getBootedUdids(): Promise | null> { const now = Date.now(); if (bootedSnapshot.booted && now - bootedSnapshot.at < 1500) { return bootedSnapshot.booted; } try { - const output = execSync("xcrun simctl list devices booted -j", { - encoding: "utf-8", - stdio: ["ignore", "pipe", "pipe"], - timeout: 3_000, + const stdout = await new Promise((resolve, reject) => { + execFile( + "xcrun", + ["simctl", "list", "devices", "booted", "-j"], + { encoding: "utf-8", timeout: 3_000 }, + (err, stdout) => { + if (err) { + reject(err); + } else { + resolve(stdout); + } + }, + ); }); - const data = JSON.parse(output) as SimctlBootedList; + const data = JSON.parse(stdout) as SimctlBootedList; const booted = new Set(); for (const runtime of Object.values(data.devices)) { for (const device of runtime) { @@ -268,7 +277,7 @@ function getPreferredDeviceUdid(): string | null { return udid; } -export function readServeSimStates(): ServeSimState[] { +export async function readServeSimStates(): Promise { let files: string[]; try { files = readdirSync(STATE_DIR).filter( @@ -277,7 +286,7 @@ export function readServeSimStates(): ServeSimState[] { } catch { return []; } - const booted = getBootedUdids(); + const booted = await getBootedUdids(); const states: ServeSimState[] = []; for (const f of files) { const path = join(STATE_DIR, f); @@ -992,28 +1001,33 @@ interface SimctlDevice { runtime: string; } -function listAllSimulators(): SimctlDevice[] { - try { - const output = execSync("xcrun simctl list devices -j", { - encoding: "utf-8", - stdio: ["ignore", "pipe", "ignore"], - timeout: 3_000, - }); - const data = JSON.parse(output) as SimctlAllList; - const out: SimctlDevice[] = []; - for (const [runtime, devices] of Object.entries(data.devices)) { - // Keep this to touch-capable simulator families that serve-sim can frame - // and inject into. tvOS is intentionally left out for now. - if (!/SimRuntime\.(iOS|watchOS|visionOS|xrOS)-/i.test(runtime)) continue; - for (const d of devices) { - if (d.isAvailable === false) continue; - out.push({ ...d, runtime: runtime.replace(/^.*SimRuntime\./, "") }); - } - } - return out; - } catch { - return []; - } +function listAllSimulators(): Promise { + return new Promise((resolve) => { + execFile( + "xcrun", + ["simctl", "list", "devices", "-j"], + { encoding: "utf-8", timeout: 3_000 }, + (err, stdout) => { + if (err) return resolve([]); + try { + const data = JSON.parse(stdout) as SimctlAllList; + const out: SimctlDevice[] = []; + for (const [runtime, devices] of Object.entries(data.devices)) { + // Keep this to touch-capable simulator families that serve-sim can + // frame and inject into. tvOS is intentionally left out for now. + if (!/SimRuntime\.(iOS|watchOS|visionOS|xrOS)-/i.test(runtime)) continue; + for (const d of devices) { + if (d.isAvailable === false) continue; + out.push({ ...d, runtime: runtime.replace(/^.*SimRuntime\./, "") }); + } + } + resolve(out); + } catch { + resolve([]); + } + }, + ); + }); } // Default per-simulator footprint when we have no running sim to measure @@ -1214,7 +1228,7 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { return { ok: true }; }; - const middleware = ((req: SimReq, res: SimRes, next?: SimNext) => { + const middleware = (async (req: SimReq, res: SimRes, next?: SimNext) => { const rawUrl: string = req.url ?? ""; const qIndex = rawUrl.indexOf("?"); const url = qIndex === -1 ? rawUrl : rawUrl.slice(0, qIndex); @@ -1237,38 +1251,36 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // inside embedded browser iframes. Serving it from the preview origin keeps // the frontend's relative assets and CSP on the local page. if (url === devtoolsFrontendBase || url.startsWith(`${devtoolsFrontendBase}/`)) { - (async () => { - const assetPath = url === devtoolsFrontendBase - ? "inspector.html" - : url.slice(devtoolsFrontendBase.length + 1); - // Reject path-traversal segments before they reach the upstream URL. - if (assetPath.split("/").some((seg) => seg === "..")) { - res.writeHead(400, { "Content-Type": "text/plain; charset=utf-8" }); - res.end("Invalid asset path"); - return; - } - try { - const upstream = await fetch( - `https://chrome-devtools-frontend.appspot.com/serve_rev/@${DEVTOOLS_FRONTEND_REV}/${assetPath}${qIndex === -1 ? "" : rawUrl.slice(qIndex)}`, - ); - const headers: Record = { - "Cache-Control": "public, max-age=604800", - }; - const contentType = upstream.headers.get("content-type"); - if (contentType) headers["Content-Type"] = contentType; - res.writeHead(upstream.status, headers); - res.end(Buffer.from(await upstream.arrayBuffer())); - } catch (err) { - res.writeHead(502, { "Content-Type": "text/plain; charset=utf-8" }); - res.end(err instanceof Error ? err.message : "Failed to load DevTools frontend"); - } - })(); + const assetPath = url === devtoolsFrontendBase + ? "inspector.html" + : url.slice(devtoolsFrontendBase.length + 1); + // Reject path-traversal segments before they reach the upstream URL. + if (assetPath.split("/").some((seg) => seg === "..")) { + res.writeHead(400, { "Content-Type": "text/plain; charset=utf-8" }); + res.end("Invalid asset path"); + return; + } + try { + const upstream = await fetch( + `https://chrome-devtools-frontend.appspot.com/serve_rev/@${DEVTOOLS_FRONTEND_REV}/${assetPath}${qIndex === -1 ? "" : rawUrl.slice(qIndex)}`, + ); + const headers: Record = { + "Cache-Control": "public, max-age=604800", + }; + const contentType = upstream.headers.get("content-type"); + if (contentType) headers["Content-Type"] = contentType; + res.writeHead(upstream.status, headers); + res.end(Buffer.from(await upstream.arrayBuffer())); + } catch (err) { + res.writeHead(502, { "Content-Type": "text/plain; charset=utf-8" }); + res.end(err instanceof Error ? err.message : "Failed to load DevTools frontend"); + } return; } // Serve the preview page if (url === base || url === base + "/") { - const states = readServeSimStates(); + const states = await readServeSimStates(); const state = selectServeSimState(states, selectedDevice); let html = loadHtml(); @@ -1320,9 +1332,9 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // Grid JSON: every supported simulator, annotated with running helper info if any. if (url === base + "/grid/api") { - const states = readServeSimStates(); + const states = await readServeSimStates(); const helperByUdid = new Map(states.map((s) => [s.device, s] as const)); - const sims = listAllSimulators(); + const sims = await listAllSimulators(); // Order mirrors Xcode's Devices window: the devices the user is actually // using float to the top — streaming first, then booted, then the // simulator they last opened in Simulator.app — and everything else falls @@ -1473,48 +1485,46 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // /devtools/page/:id on localhost; the preview adds iframe-safe frontend // URLs so the browser UI can embed Chrome DevTools. if (url === base + "/devtools") { - (async () => { - const states = readServeSimStates(); - const state = selectServeSimState(states, selectedDevice); - if (!state) { - res.writeHead(404, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ error: "No serve-sim device" })); - return; - } - try { - const bridge = await getInspectWebKitBridge(); - const bridgeTargets = await bridge.listTargets(); - // Proxy mode routes the inspector socket through the preview's - // same-origin `/devtools` proxy; otherwise the browser talks to the - // bridge's loopback port directly (the pre-proxy behavior). - const wsProtocol = proxyHelpers ? websocketProtocolForRequest(req) : "ws"; - const wsTargetBase = proxyHelpers - ? `${hostForRequest(req) ?? `127.0.0.1:${bridge.port}`}${devtoolsPrefix}` - : `127.0.0.1:${bridge.port}/devtools`; - // inspect-webkit@0.0.3 only exposes `sim:` for - // simulator targets, which can't be reconciled against a sim UDID. - // Surface every booted sim's targets (Safari Develop-menu behavior) - // until inspect-webkit grows a real UDID we can filter on. - const targets = bridgeTargets.map((target) => ({ - ...target, - webSocketDebuggerUrl: `${wsProtocol}://${wsTargetBase}/page/${encodeURIComponent(target.id)}`, - devtoolsFrontendUrl: devtoolsFrontendUrl(devtoolsFrontendBase, wsProtocol, wsTargetBase, target.id), - })); - res.writeHead(200, { - "Content-Type": "application/json", - "Cache-Control": "no-store", - }); - res.end(JSON.stringify({ - port: bridge.port, - targets, - })); - } catch (err) { - res.writeHead(500, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ - error: err instanceof Error ? err.message : "Failed to start inspect-webkit", - })); - } - })(); + const states = await readServeSimStates(); + const state = selectServeSimState(states, selectedDevice); + if (!state) { + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "No serve-sim device" })); + return; + } + try { + const bridge = await getInspectWebKitBridge(); + const bridgeTargets = await bridge.listTargets(); + // Proxy mode routes the inspector socket through the preview's + // same-origin `/devtools` proxy; otherwise the browser talks to the + // bridge's loopback port directly (the pre-proxy behavior). + const wsProtocol = proxyHelpers ? websocketProtocolForRequest(req) : "ws"; + const wsTargetBase = proxyHelpers + ? `${hostForRequest(req) ?? `127.0.0.1:${bridge.port}`}${devtoolsPrefix}` + : `127.0.0.1:${bridge.port}/devtools`; + // inspect-webkit@0.0.3 only exposes `sim:` for + // simulator targets, which can't be reconciled against a sim UDID. + // Surface every booted sim's targets (Safari Develop-menu behavior) + // until inspect-webkit grows a real UDID we can filter on. + const targets = bridgeTargets.map((target) => ({ + ...target, + webSocketDebuggerUrl: `${wsProtocol}://${wsTargetBase}/page/${encodeURIComponent(target.id)}`, + devtoolsFrontendUrl: devtoolsFrontendUrl(devtoolsFrontendBase, wsProtocol, wsTargetBase, target.id), + })); + res.writeHead(200, { + "Content-Type": "application/json", + "Cache-Control": "no-store", + }); + res.end(JSON.stringify({ + port: bridge.port, + targets, + })); + } catch (err) { + res.writeHead(500, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ + error: err instanceof Error ? err.message : "Failed to start inspect-webkit", + })); + } return; } @@ -1576,7 +1586,7 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // JSON API: serve-sim state if (url === base + "/api") { - const states = readServeSimStates(); + const states = await readServeSimStates(); const state = selectServeSimState(states, selectedDevice); // The web UI polls /api every ~2s, so logging every hit floods the // debug stream with identical lines. Only log when the selection @@ -1607,8 +1617,8 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // or the device selection changes, so we watch the state dir and emit only // on change instead of re-sending identical JSON on a fixed interval. if (url === base + "/api/events") { - const computeConfig = (): string => { - const states = readServeSimStates(); + const computeConfig = async (): Promise => { + const states = await readServeSimStates(); const state = selectServeSimState(states, selectedDevice); const remoteState = state ? rewriteStateForRequestHost(state, hostForRequest(req), base, httpProtocolForRequest(req), proxyHelpers) : null; return JSON.stringify( @@ -1624,13 +1634,13 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { }); res.write(":\n\n"); - let lastSent = computeConfig(); + let lastSent = await computeConfig(); res.write("data: " + lastSent + "\n\n"); let closed = false; - const sendIfChanged = () => { + const sendIfChanged = async () => { if (closed || res.writableEnded) return; - const next = computeConfig(); + const next = await computeConfig(); if (next === lastSent) return; lastSent = next; res.write("data: " + next + "\n\n"); @@ -1690,7 +1700,7 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // SSE: normalized accessibility snapshot stream if (url === base + "/ax") { - const states = readServeSimStates(); + const states = await readServeSimStates(); const state = selectServeSimState(states, selectedDevice); if (!state) { res.writeHead(404); @@ -1792,7 +1802,7 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // log line. Filtering is done here (not in the browser) so the SSE stream // stays narrow and the client can listen without rate-limit concerns. if (url === base + "/appstate") { - const states = readServeSimStates(); + const states = await readServeSimStates(); const state = selectServeSimState(states, selectedDevice); if (!state) { res.writeHead(404); @@ -1814,19 +1824,17 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { // the helper's AX bridge for the current frontmost app via // `proc_pidpath`+Info.plist resolution and emit it before tailing. let lastBundle = ""; - void (async () => { - try { - const info = JSON.parse(await axFrontmostAsync(udid)) as { bundleId?: string; pid?: number }; - if (!info.bundleId || !isUserFacingBundle(info.bundleId)) return; - if (res.writableEnded) return; - lastBundle = info.bundleId; - const isReactNative = await detectReactNative(udid, info.bundleId); - if (res.writableEnded) return; - res.write("data: " + JSON.stringify({ bundleId: info.bundleId, pid: info.pid, isReactNative }) + "\n\n"); - } catch { - // AX bridge may be warming up — the log tail fills in once anything moves. - } - })(); + try { + const info = JSON.parse(await axFrontmostAsync(udid)) as { bundleId?: string; pid?: number }; + if (!info.bundleId || !isUserFacingBundle(info.bundleId)) return; + if (res.writableEnded) return; + lastBundle = info.bundleId; + const isReactNative = await detectReactNative(udid, info.bundleId); + if (res.writableEnded) return; + res.write("data: " + JSON.stringify({ bundleId: info.bundleId, pid: info.pid, isReactNative }) + "\n\n"); + } catch { + // AX bridge may be warming up — the log tail fills in once anything moves. + } const child: ChildProcess = spawn("xcrun", [ "simctl", "spawn", udid, "log", "stream", @@ -1879,7 +1887,7 @@ export function simMiddleware(options?: SimMiddlewareOptions): SimMiddleware { } // Not ours — pass through - if (next) next(); + if (next) return next(); }) as SimMiddleware; middleware.handleUpgrade = (req: SimReq, socket: Socket, head: Buffer) => { const rawUrl = req.url ?? ""; diff --git a/packages/serve-sim/src/native.ts b/packages/serve-sim/src/native.ts index 4bce14181..ec4d60efa 100644 --- a/packages/serve-sim/src/native.ts +++ b/packages/serve-sim/src/native.ts @@ -19,53 +19,58 @@ const require = createRequire(import.meta.url); // handle is garbage-collected (Swift `deinit`), so there are no explicit // destroy/free calls here. interface SimHIDHandle { - touch(type: TouchType, x: number, y: number, w: number, hh: number, edge: number): void; - multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, hh: number): void; - button(button: string): void; - buttonHid(page: number, usage: number, phase: ButtonPhase): void; - key(type: KeyType, usage: number): void; - scroll(dx: number, dy: number, anchorX: number, anchorY: number, w: number, hh: number): void; - digitalCrown(delta: number): void; - orientation(orientation: number): boolean; - memoryWarning(): void; - softwareKeyboard(): void; - caDebug(name: string, enabled: boolean): boolean; + touch(type: TouchType, x: number, y: number, w: number, hh: number, edge: number): Promise; + multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, hh: number): Promise; + button(button: string): Promise; + buttonHid(page: number, usage: number, phase: ButtonPhase): Promise; + key(type: KeyType, usage: number): Promise; + scroll(dx: number, dy: number, anchorX: number, anchorY: number, w: number, hh: number): Promise; + digitalCrown(delta: number): Promise; + orientation(orientation: number): Promise; + memoryWarning(): Promise; + softwareKeyboard(): Promise; + caDebug(name: string, enabled: boolean): Promise; } interface SimCaptureHandle { start(): void; - setAvccActive(active: boolean): void; - requestKeyframe(): void; - screenSize(): { width: number; height: number }; stop(): void; + subscribe(codec: number, onFrame: RawFrameCallback): Promise<() => void>; } interface NativeAddon { SimHID: new (udid: string) => SimHIDHandle; - SimCapture: new (udid: string, onFrame: RawFrameCallback) => SimCaptureHandle; + SimCapture: new (udid: string) => SimCaptureHandle; axDescribe(udid: string): Promise; axFrontmost(udid: string): Promise; } // (codec, data, width, height, flags) — codec 0=MJPEG 1=AVCC; flags bit0=desc bit1=keyframe. -type RawFrameCallback = (codec: number, data: Buffer, width: number, height: number, flags: number) => void; - +type RawFrameCallback = ( + data: Uint8Array, + width: number, + height: number, + flags: number, +) => Promise; + +const CODEC_MJPEG = 0; const CODEC_AVCC = 1; const FLAG_DESCRIPTION = 1 << 0; const FLAG_KEYFRAME = 1 << 1; -export interface NativeFrame { - /** `mjpeg` = a full JPEG; `avcc` = a length-prefixed AVCC envelope chunk. */ - codec: "mjpeg" | "avcc"; - /** Encoded bytes, ready to write to the stream wire. */ - data: Buffer; +export type MjpegFrame = { + data: Uint8Array; + width: number; + height: number; +}; + +export type AvccFrame = { + data: Uint8Array; width: number; height: number; - /** AVCC only: this chunk is the avcC parameter-set blob (decoder config). */ isDescription: boolean; - /** AVCC only: this chunk is an IDR keyframe (a decoder can start here). */ isKeyframe: boolean; -} +}; export type TouchType = "begin" | "move" | "end"; export type KeyType = "down" | "up"; @@ -123,81 +128,72 @@ export class NativeHid { // mid-gesture, the guest is left with a stuck finger that wedges input until // the sim reboots. The spawned helper used to absorb this in its own process; // `guard` restores that isolation by swallowing malformed-input errors. - private guard(op: string, fn: () => T, fallback: T): T { + private async guard(op: string, fn: () => PromiseLike, fallback: T): Promise { try { - return fn(); + return await fn(); } catch (err) { console.error(`[hid] ${op} ignored bad input:`, err instanceof Error ? err.message : err); return fallback; } } - touch(type: TouchType, x: number, y: number, w: number, h: number, edge = 0): void { - this.guard("touch", () => this.handle.touch(type, x, y, w, h, edge), undefined); + touch(type: TouchType, x: number, y: number, w: number, h: number, edge = 0): Promise { + return this.guard("touch", () => this.handle.touch(type, x, y, w, h, edge), undefined); } - multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, h: number): void { - this.guard("multiTouch", () => this.handle.multiTouch(type, x1, y1, x2, y2, w, h), undefined); + multiTouch(type: TouchType, x1: number, y1: number, x2: number, y2: number, w: number, h: number): Promise { + return this.guard("multiTouch", () => this.handle.multiTouch(type, x1, y1, x2, y2, w, h), undefined); } - button(button: string): void { - this.guard("button", () => this.handle.button(button), undefined); + button(button: string): Promise { + return this.guard("button", () => this.handle.button(button), undefined); } - buttonHid(page: number, usage: number, phase: ButtonPhase = "press"): void { - this.guard("buttonHid", () => this.handle.buttonHid(page, usage, phase), undefined); + buttonHid(page: number, usage: number, phase: ButtonPhase = "press"): Promise { + return this.guard("buttonHid", () => this.handle.buttonHid(page, usage, phase), undefined); } - key(type: KeyType, usage: number): void { - this.guard("key", () => this.handle.key(type, usage), undefined); + key(type: KeyType, usage: number): Promise { + return this.guard("key", () => this.handle.key(type, usage), undefined); } /** anchorX/anchorY default to screen center when omitted. */ - scroll(dx: number, dy: number, w: number, h: number, anchorX?: number, anchorY?: number): void { - this.guard("scroll", () => this.handle.scroll(dx, dy, anchorX ?? NaN, anchorY ?? NaN, w, h), undefined); + scroll(dx: number, dy: number, w: number, h: number, anchorX?: number, anchorY?: number): Promise { + return this.guard("scroll", () => this.handle.scroll(dx, dy, anchorX ?? NaN, anchorY ?? NaN, w, h), undefined); } - digitalCrown(delta: number): void { - this.guard("digitalCrown", () => this.handle.digitalCrown(delta), undefined); + digitalCrown(delta: number): Promise { + return this.guard("digitalCrown", () => this.handle.digitalCrown(delta), undefined); } - orientation(orientation: number): boolean { + orientation(orientation: number): Promise { return this.guard("orientation", () => this.handle.orientation(orientation), false); } - memoryWarning(): void { - this.guard("memoryWarning", () => this.handle.memoryWarning(), undefined); + memoryWarning(): Promise { + return this.guard("memoryWarning", () => this.handle.memoryWarning(), undefined); } - softwareKeyboard(): void { - this.guard("softwareKeyboard", () => this.handle.softwareKeyboard(), undefined); + softwareKeyboard(): Promise { + return this.guard("softwareKeyboard", () => this.handle.softwareKeyboard(), undefined); } - caDebug(name: string, enabled: boolean): boolean { + caDebug(name: string, enabled: boolean): Promise { return this.guard("caDebug", () => this.handle.caDebug(name, enabled), false); } } /** * In-process frame capture + encode for one simulator. Replaces the spawned - * helper's capture pipeline: MJPEG frames are always produced; H.264/AVCC runs - * only while `setAvccActive(true)`. Encoded frames arrive via the `onFrame` - * callback on the JS thread (marshalled from the native encode thread). + * helper's capture pipeline. MJPEG and H.264/AVCC frames are produced while + * callers hold codec-specific subscriptions; encoded frames arrive on the JS + * thread after being marshalled from the native encode thread. */ export class NativeCapture { private readonly handle: SimCaptureHandle; - constructor(udid: string, onFrame: (frame: NativeFrame) => void) { - this.handle = new (load().SimCapture)(udid, (codec, data, width, height, flags) => { - onFrame({ - codec: codec === CODEC_AVCC ? "avcc" : "mjpeg", - data, - width, - height, - isDescription: (flags & FLAG_DESCRIPTION) !== 0, - isKeyframe: (flags & FLAG_KEYFRAME) !== 0, - }); - }); + constructor(udid: string) { + this.handle = new (load().SimCapture)(udid); } /** Begin capturing. Throws if the device isn't booted. */ @@ -205,18 +201,22 @@ export class NativeCapture { this.handle.start(); } - /** Enable/disable H.264 encoding (forces an IDR on the next frame when enabled). */ - setAvccActive(active: boolean): void { - this.handle.setAvccActive(active); - } - - /** Force the next H.264 frame to a keyframe (e.g. when a new AVCC viewer joins). */ - requestKeyframe(): void { - this.handle.requestKeyframe(); + subscribeMjpeg(onFrame: (frame: MjpegFrame) => Promise): Promise<() => void> { + return this.handle.subscribe(CODEC_MJPEG, (data, width, height, _flags) => { + return onFrame({ data, width, height }); + }); } - screenSize(): { width: number; height: number } { - return this.handle.screenSize(); + subscribeAvcc(onFrame: (frame: AvccFrame) => Promise): Promise<() => void> { + return this.handle.subscribe(CODEC_AVCC, (data, width, height, flags) => { + return onFrame({ + data, + width, + height, + isDescription: (flags & FLAG_DESCRIPTION) !== 0, + isKeyframe: (flags & FLAG_KEYFRAME) !== 0, + }); + }); } /** Halt frame production. Full teardown happens when this object is GC'd. */ diff --git a/packages/serve-sim/src/runtime.ts b/packages/serve-sim/src/runtime.ts index 0663ba09f..813da964a 100644 --- a/packages/serve-sim/src/runtime.ts +++ b/packages/serve-sim/src/runtime.ts @@ -32,8 +32,8 @@ export interface PreviewServer { type ConnectMiddleware = ( req: IncomingMessage, res: ServerResponse, - next: () => void, -) => void; + next: () => Promise, +) => Promise; type PreviewMiddleware = ConnectMiddleware & { handleUpgrade?: (req: IncomingMessage, socket: Socket, head: Buffer) => void; @@ -146,12 +146,29 @@ export async function servePreview(opts: { */ host?: string; }): Promise { - const internalServer = createHttpServer((req, res) => { - opts.middleware(req, res, () => { - if (!res.headersSent) res.statusCode = 404; - res.end("Not found"); - }); - }); + const isBun = !!process.versions.bun + + const internalServer = createHttpServer( + { + highWaterMark: 1024 * 1024 * 5, + }, + async (req, res) => { + try { + return await opts.middleware(req, res, async () => { + if (!res.headersSent) res.statusCode = 404; + res.end("Not found"); + }); + } catch (err) { + console.error("Middleware error:", err); + if (!res.headersSent) { + res.statusCode = 500; + res.end("Internal Server Error"); + } else { + res.end(); + } + } + } + ); // MJPEG streams + SSE log channel are long-lived; clear the default 2-min // socket timeout so they don't get torn down mid-stream. internalServer.keepAliveTimeout = 0; @@ -177,7 +194,11 @@ export async function servePreview(opts: { }; internalServer.once("error", onError); internalServer.once("listening", onListening); - internalServer.listen(0, "127.0.0.1"); + if (isBun) { + internalServer.listen(0, "127.0.0.1"); + } else { + internalServer.listen(opts.port, opts.host ?? "127.0.0.1"); + } }); const internalAddress = internalServer.address(); @@ -186,29 +207,34 @@ export async function servePreview(opts: { throw new Error("Failed to bind preview HTTP server"); } - const frontServer = createPreviewFrontServer(opts.middleware, internalAddress.port); - - await new Promise((resolve, reject) => { - const onError = (err: Error & { code?: string }) => { - frontServer.removeListener("listening", onListening); - // The internal server is already listening; if the front fails to bind - // (e.g. EADDRINUSE during the port-scan retry loop), close it too so we - // don't leak a listener per attempt. - internalServer.close(() => reject(err)); - }; - const onListening = () => { - frontServer.removeListener("error", onError); - resolve(); - }; - frontServer.once("error", onError); - frontServer.once("listening", onListening); - frontServer.listen(opts.port, opts.host ?? "127.0.0.1"); - }); + let maybeFrontServer: NetServer | undefined; + if (isBun) { + // works around a bug where Bun fails to proxy websockets + // https://github.com/oven-sh/bun/issues/14522 + const frontServer = createPreviewFrontServer(opts.middleware, internalAddress.port); + maybeFrontServer = frontServer; + await new Promise((resolve, reject) => { + const onError = (err: Error & { code?: string }) => { + frontServer.removeListener("listening", onListening); + // The internal server is already listening; if the front fails to bind + // (e.g. EADDRINUSE during the port-scan retry loop), close it too so we + // don't leak a listener per attempt. + internalServer.close(() => reject(err)); + }; + const onListening = () => { + frontServer.removeListener("error", onError); + resolve(); + }; + frontServer.once("error", onError); + frontServer.once("listening", onListening); + frontServer.listen(opts.port, opts.host ?? "127.0.0.1"); + }); + } return { stop: () => { - frontServer.close(); - internalServer.close(); + internalServer.close() + maybeFrontServer?.close() }, }; }