diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7ac403c..8db94e4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -12,19 +12,6 @@ permissions: contents: read jobs: - lint: - runs-on: macos-latest - - steps: - - name: Checkout code - uses: actions/checkout@v5 - - - name: Install SwiftLint - run: brew install swiftlint - - - name: Run lint - run: make lint - trivy: runs-on: macos-latest @@ -42,7 +29,7 @@ jobs: run: trivy fs --severity HIGH,CRITICAL --exit-code 1 . build-ipa: - needs: [lint, trivy] + needs: [trivy] runs-on: macos-latest steps: @@ -65,7 +52,7 @@ jobs: retention-days: 4 build-sim: - needs: [lint, trivy] + needs: [trivy] runs-on: macos-latest steps: diff --git a/.gitmodules b/.gitmodules index 0e86687..e69de29 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +0,0 @@ -[submodule "opus-codec/Sources/Copus"] - path = opus-codec/Sources/Copus - url = https://github.com/xiph/opus.git diff --git a/.swiftlint.yml b/.swiftlint.yml index 331539b..39bc5d5 100644 --- a/.swiftlint.yml +++ b/.swiftlint.yml @@ -2,11 +2,8 @@ # Paths to include/exclude included: - - devicekit-ios - - BroadcastUploadExtension - - BroadcastUploadExtensionSetupUI - - h264-codec - - tcp + - DeviceKit + - DeviceKitTests excluded: - Pods diff --git a/BroadcastUploadExtension/CGRect+Extension.swift b/BroadcastUploadExtension/CGRect+Extension.swift deleted file mode 100644 index 79bc777..0000000 --- a/BroadcastUploadExtension/CGRect+Extension.swift +++ /dev/null @@ -1,26 +0,0 @@ -import UIKit - -extension CGRect { - static var logicalResolutionScreen: CGRect { - makeScreen(bounds: UIScreen.main.bounds) - } - - static var actualResolutionScreen: CGRect { - makeScreen(bounds: UIScreen.main.nativeBounds) - } - - private static func makeScreen(bounds: CGRect) -> CGRect { - let size = CGSize(width: bounds.width, height: bounds.height) - return CGRect(origin: .zero, size: size) - } - - func scaledDimensions(_ factor: Float) -> (width: Int32, height: Int32) { - let scaledWidth = Int32(Float(width) * factor) - let scaledHeight = Int32(Float(height) * factor) - return (width: scaledWidth, height: scaledHeight) - } - - private var nativeSide: Float { - Float(max(height, width)) - } -} diff --git a/BroadcastUploadExtension/CMSampleBuffer+Extension.swift b/BroadcastUploadExtension/CMSampleBuffer+Extension.swift deleted file mode 100644 index 7357bc2..0000000 --- a/BroadcastUploadExtension/CMSampleBuffer+Extension.swift +++ /dev/null @@ -1,20 +0,0 @@ -import CoreMedia -import ReplayKit - -extension CMSampleBuffer { - var orientation: CGImagePropertyOrientation? { - guard - let sampleOrientation = CMGetAttachment( - self, - key: RPVideoSampleOrientationKey as CFString, - attachmentModeOut: nil - ) - else { - return nil - } - - return CGImagePropertyOrientation( - rawValue: sampleOrientation.uint32Value - ) - } -} diff --git a/BroadcastUploadExtension/Info.plist b/BroadcastUploadExtension/Info.plist deleted file mode 100644 index e936790..0000000 --- a/BroadcastUploadExtension/Info.plist +++ /dev/null @@ -1,15 +0,0 @@ - - - - - NSExtension - - NSExtensionPointIdentifier - com.apple.broadcast-services-upload - NSExtensionPrincipalClass - $(PRODUCT_MODULE_NAME).SampleHandler - RPBroadcastProcessMode - RPBroadcastProcessModeSampleBuffer - - - diff --git a/BroadcastUploadExtension/SampleHandler.swift b/BroadcastUploadExtension/SampleHandler.swift deleted file mode 100644 index 102c874..0000000 --- a/BroadcastUploadExtension/SampleHandler.swift +++ /dev/null @@ -1,88 +0,0 @@ -import Foundation -import ReplayKit - -class SampleHandler: RPBroadcastSampleHandler { - private static let defaultPort: UInt16 = 12005 - private static let defaultScaleFactor: Float = 0.5 - private static let defaultQualityFactor: Float = 0.8 - private static let defaultExpectedFrameRate: Int = 30 - private static let defaultAverageBitRate: Int = 8_000_000 - private static let defaultAudioPort: UInt16 = 12006 - private static let defaultAudioBitRate: Int = 64_000 - - private static let rpcPort: UInt16 = 12004 - - private var context: CIContext? - private var screenStreamer: ScreenStreamer? - - override func broadcastStarted(withSetupInfo setupInfo: [String: NSObject]?) { - let port = setupInfo?["port"] as? UInt16 ?? Self.defaultPort - let usesActualResolution = setupInfo?["usesActualResolution"] as? Bool ?? true - let rect: CGRect = usesActualResolution ? .actualResolutionScreen : .logicalResolutionScreen - let scaleFactor = setupInfo?["scaleFactor"] as? Float ?? Self.defaultScaleFactor - let qualityFactor = setupInfo?["qualityFactor"] as? Float ?? Self.defaultQualityFactor - let expectedFrameRate = setupInfo?["expectedFrameRate"] as? Int ?? Self.defaultExpectedFrameRate - let averageBitRate = setupInfo?["averageBitRate"] as? Int ?? Self.defaultAverageBitRate - let isRealTime = setupInfo?["isRealTime"] as? Bool ?? false - let audioEnabled = setupInfo?["audioEnabled"] as? Bool ?? true - let audioPort = setupInfo?["audioPort"] as? UInt16 ?? Self.defaultAudioPort - let audioBitRate = setupInfo?["audioBitRate"] as? Int ?? Self.defaultAudioBitRate - - context = CIContext() - screenStreamer = ScreenStreamer() - - do { - try screenStreamer?.start(StreamerConfig( - port: port, - rect: rect, - scaleFactor: scaleFactor, - qualityFactor: qualityFactor, - expectedFrameRate: expectedFrameRate, - averageBitRate: averageBitRate, - isRealTime: isRealTime, - audioPort: audioEnabled ? audioPort : nil, - audioBitRate: audioBitRate - )) - } catch { - fatalError(error.localizedDescription) - } - } - - override func broadcastPaused() { - } - - override func broadcastResumed() { - } - - override func broadcastFinished() { - screenStreamer?.stop() - context?.clearCaches() - } - - override func processSampleBuffer( - _ sampleBuffer: CMSampleBuffer, - with sampleBufferType: RPSampleBufferType - ) { - switch sampleBufferType { - - case .video: - guard let context = context else { return } - guard let orientation = sampleBuffer.orientation else { return } - - screenStreamer?.encode( - sampleBuffer: sampleBuffer, - context: context, - orientation: orientation - ) - - case .audioApp: - screenStreamer?.encodeAudio(sampleBuffer: sampleBuffer) - - case .audioMic: - break - - @unknown default: - fatalError("Unknown type of sample buffer") - } - } -} diff --git a/BroadcastUploadExtension/ScreenStreamer.swift b/BroadcastUploadExtension/ScreenStreamer.swift deleted file mode 100644 index a924208..0000000 --- a/BroadcastUploadExtension/ScreenStreamer.swift +++ /dev/null @@ -1,215 +0,0 @@ -import CoreImage -import CoreMedia -import H264Codec -import OpusCodec - -struct StreamerConfig { - let port: UInt16 - let rect: CGRect - let scaleFactor: Float - let qualityFactor: Float - let expectedFrameRate: Int - let averageBitRate: Int - let isRealTime: Bool - let audioPort: UInt16? - let audioBitRate: Int -} - -final class ScreenStreamer { - private let h264Encoder: H264Encoder - private let tcpServer: TCPServer - private let audioEncoder: OpusAudioEncoder - private let audioServer: TCPServer - - private var messageBuffer = Data() - private var isPaused = false - private var isStopped = false - private var loggedMissingAudioClient = false - - init( - videoEncoder: H264Encoder = H264Encoder(), - tcpServer: TCPServer = TCPServer(), - audioEncoder: OpusAudioEncoder = OpusAudioEncoder(), - audioServer: TCPServer = TCPServer() - ) { - self.h264Encoder = videoEncoder - self.tcpServer = tcpServer - self.audioEncoder = audioEncoder - self.audioServer = audioServer - } - - func start(_ config: StreamerConfig) throws { - isPaused = false - isStopped = false - - try tcpServer.start(port: config.port) - - let dimensions = config.rect.scaledDimensions(config.scaleFactor) - try h264Encoder.configureCompressSession(H264EncoderConfig( - width: dimensions.width, - height: dimensions.height, - isRealTime: config.isRealTime, - expectedFrameRate: config.expectedFrameRate, - averageBitRate: config.averageBitRate, - quality: config.qualityFactor - )) - - h264Encoder.naluHandling = { [weak self] data in - guard let self else { return } - tcpServer.dataHandler?(data) - } - - if let audioPort = config.audioPort { - audioEncoder.updateBitRate(config.audioBitRate) - try audioServer.start(port: audioPort) - audioEncoder.opusHandling = { [weak self] data in - guard let self else { return } - guard let dataHandler = audioServer.dataHandler else { - if !self.loggedMissingAudioClient { - self.loggedMissingAudioClient = true - NSLog("[ScreenStreamer] Opus frame ready but no audio client connected") - } - return - } - dataHandler(self.lengthPrefixed(data)) - } - } else { - audioEncoder.opusHandling = nil - audioServer.stop() - } - - tcpServer.messageHandler = { [weak self] data in - guard let self else { return } - self.handleIncomingData(data) - } - } - - private func handleIncomingData(_ data: Data) { - messageBuffer.append(data) - - while messageBuffer.count >= 4 { - let lengthBytes = messageBuffer.prefix(4) - let length = Int(UInt32(bigEndian: lengthBytes.withUnsafeBytes { $0.load(as: UInt32.self) })) - - guard messageBuffer.count >= 4 + length else { break } - - let messageData = messageBuffer.subdata(in: 4..<(4 + length)) - messageBuffer.removeFirst(4 + length) - - handleJSONRPC(messageData) - } - } - - private func handleJSONRPC(_ data: Data) { - guard let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], - let method = json["method"] as? String else { - print("[ScreenStreamer] Invalid JSON-RPC message") - return - } - - switch method { - case "screencapture.setConfiguration": - handleSetConfiguration(params: json["params"] as? [String: Any]) - case "screencapture.pause": - handlePause() - case "screencapture.resume": - handleResume() - case "screencapture.stop": - handleStop() - default: - print("[ScreenStreamer] Unknown method: \(method)") - } - } - - private func handleSetConfiguration(params: [String: Any]?) { - guard let params = params, - let bitrate = params["bitrate"] as? Int else { - print("[ScreenStreamer] Invalid params for setConfiguration") - return - } - - let frameRate = params["frameRate"] as? Int - - guard bitrate >= 100_000 && bitrate <= 8_000_000 else { - print("[ScreenStreamer] Bitrate out of range: \(bitrate) (must be 100000-8000000)") - return - } - - if let fr = frameRate, fr < 1 || fr > 60 { - print("[ScreenStreamer] Frame rate out of range: \(fr) (must be 1-60)") - return - } - - do { - try h264Encoder.updateEncoderSettings(newBitrate: bitrate, newFrameRate: frameRate) - print("[ScreenStreamer] ✓ Configuration updated: bitrate=\(bitrate) bps" + - (frameRate.map { ", frameRate=\($0)" } ?? "")) - } catch { - print("[ScreenStreamer] ✗ Failed to update encoder: \(error)") - } - } - - private func handlePause() { - isPaused = true - print("[ScreenStreamer] ✓ Paused") - } - - private func handleResume() { - isPaused = false - print("[ScreenStreamer] ✓ Resumed") - } - - private func handleStop() { - stop() - print("[ScreenStreamer] ✓ Stopped") - } - - func encode( - sampleBuffer: CMSampleBuffer, - context: CIContext, - orientation: CGImagePropertyOrientation - ) { - guard !isPaused, !isStopped else { return } - h264Encoder.encode( - sampleBuffer: sampleBuffer, - context: context, - orientation: orientation - ) - } - - func encode( - imageBuffer: CVImageBuffer, - timestamp: CMTime, - context: CIContext, - orientation: CGImagePropertyOrientation - ) { - guard !isPaused, !isStopped else { return } - h264Encoder.encode( - imageBuffer: imageBuffer, - timestamp: timestamp, - context: context, - orientation: orientation - ) - } - - func encodeAudio(sampleBuffer: CMSampleBuffer) { - guard !isPaused, !isStopped else { return } - audioEncoder.encode(sampleBuffer: sampleBuffer) - } - - func stop() { - isStopped = true - tcpServer.stop() - h264Encoder.invalidateCompressionSession() - audioServer.stop() - audioEncoder.invalidate() - } - - private func lengthPrefixed(_ data: Data) -> Data { - var length = UInt32(data.count).bigEndian - var packet = Data() - packet.append(Data(bytes: &length, count: MemoryLayout.size(ofValue: length))) - packet.append(data) - return packet - } -} diff --git a/BroadcastUploadExtension/TCPServer.swift b/BroadcastUploadExtension/TCPServer.swift deleted file mode 100644 index fba39cc..0000000 --- a/BroadcastUploadExtension/TCPServer.swift +++ /dev/null @@ -1,101 +0,0 @@ -import Foundation -import Network - -public final class TCPServer { - - enum ServerError: Error { - case invalidPortNumber - } - - private lazy var listeningQueue = DispatchQueue(label: "tcp.server.queue") - private lazy var connectionQueue = DispatchQueue(label: "tcp.connection.queue") - - private var listener: NWListener? - - public var dataHandler: ((Data) -> Void)? - public var messageHandler: ((Data) -> Void)? - public var onClientConnected: (() -> Void)? - - public init() {} - - public func start(port: UInt16, host: String = "127.0.0.1") throws { - listener?.cancel() - - guard let port = NWEndpoint.Port(rawValue: port) else { - throw ServerError.invalidPortNumber - } - - let params = NWParameters.tcp - params.requiredLocalEndpoint = NWEndpoint.hostPort( - host: NWEndpoint.Host(host), - port: port - ) - listener = try NWListener(using: params) - - listener?.stateUpdateHandler = { state in - if state == .ready { - print("listener is ready to receive data") - } - } - - listener?.newConnectionHandler = { [weak self] connection in - guard let weakSelf = self else { return } - - print("connection requested --> \(connection.endpoint)") - - connection.stateUpdateHandler = { state in - if state == .ready { - weakSelf.dataHandler = { data in - weakSelf.send(data: data, on: connection) - } - - weakSelf.startReceiving(on: connection) - - weakSelf.onClientConnected?() - } - } - - connection.start(queue: weakSelf.connectionQueue) - } - - listener?.start(queue: listeningQueue) - } - - public func stop() { - listener?.cancel() - listener = nil - dataHandler = nil - messageHandler = nil - onClientConnected = nil - } - - private func startReceiving(on connection: NWConnection) { - connection.receive(minimumIncompleteLength: 1, maximumLength: 65536) { [weak self] data, _, isComplete, error in - guard let self = self else { return } - - if let error = error { - print("[TCPServer] Receive error: \(error)") - return - } - - if let data = data, !data.isEmpty { - self.messageHandler?(data) - } - - if !isComplete { - self.startReceiving(on: connection) - } - } - } - - private func send(data: Data, on connection: NWConnection) { - connection.send( - content: data, - completion: .contentProcessed { error in - if let error = error { - print(error) - } - } - ) - } -} diff --git a/DeviceKitTests/H264Stream/ScreenshotH264Stream.swift b/DeviceKitTests/H264Stream/ScreenshotH264Stream.swift deleted file mode 100644 index 76f63ae..0000000 --- a/DeviceKitTests/H264Stream/ScreenshotH264Stream.swift +++ /dev/null @@ -1,354 +0,0 @@ -import Foundation -import CoreImage -import CoreMedia -import CoreVideo -import H264Codec -import TCP -import os -import ImageIO -import Accelerate - -/// Configuration for the H.264 screenshot stream. -struct ScreenshotStreamConfig { - /// Target frames per second. - var fps: Int = 15 - - /// Target bitrate in bits per second. - var bitrate: Int = 2_000_000 - - /// JPEG quality for screenshot capture (0.0-1.0). - var screenshotQuality: Double = 0.5 - - /// H.264 encoder quality hint (0.0-1.0). - var encoderQuality: Float = 0.5 - - /// Scale factor (0.1-1.0). 1.0 = full resolution. - var scale: CGFloat = 1.0 - - /// TCP port for streaming. - var port: UInt16 = 12007 -} - -/// Streams H.264 encoded video from screenshots over TCP. -/// -/// Similar to ScreenStreamer but captures screenshots instead of ReplayKit frames. -/// Simply forwards NAL units to TCP - no caching or duplicate detection. -@MainActor -final class ScreenshotH264Stream { - - private let logger = Logger( - subsystem: Bundle.main.bundleIdentifier ?? "devicekit-ios", - category: "H264Stream" - ) - - private var config = ScreenshotStreamConfig() - private var encoder: H264Encoder? - private var tcpServer: TCPServer? - private var ciContext: CIContext? - private var streamTask: Task? - private var frameNumber: Int64 = 0 - private var isRunning = false - private var pixelBufferPool: CVPixelBufferPool? - - // Cache SPS/PPS/IDR for late-connecting clients - private var cachedSPS: Data? - private var cachedPPS: Data? - private var cachedIDR: Data? - - private let processingQueue = DispatchQueue( - label: "h264.screenshot.processing", - qos: .userInitiated - ) - - // Latency tracking - private var frameStartTime: UInt64 = 0 - - /// Starts the H.264 stream. - func start(config: ScreenshotStreamConfig = ScreenshotStreamConfig()) throws { - guard !isRunning else { - logger.warning("Stream already running") - return - } - - self.config = config - logger.info("Starting H264 stream: \(config.fps)fps, \(config.bitrate)bps, port=\(config.port)") - - // Initialize Core Image context (GPU-backed) - ciContext = CIContext(options: [.useSoftwareRenderer: false]) - - // Initialize encoder - encoder = H264Encoder() - - let screenSize = getScreenSize() - let width = Int32(screenSize.width * config.scale) - let height = Int32(screenSize.height * config.scale) - - logger.info("Encoder dimensions: \(width)x\(height)") - - // Create pixel buffer pool for reuse (avoids allocation per frame) - let poolAttrs: [CFString: Any] = [ - kCVPixelBufferPoolMinimumBufferCountKey: 3 - ] - let pixelBufferAttrs: [CFString: Any] = [ - kCVPixelBufferWidthKey: width, - kCVPixelBufferHeightKey: height, - kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_32BGRA, - kCVPixelBufferIOSurfacePropertiesKey: [:], // IOSurface-backed for GPU - kCVPixelBufferMetalCompatibilityKey: true - ] - CVPixelBufferPoolCreate( - kCFAllocatorDefault, - poolAttrs as CFDictionary, - pixelBufferAttrs as CFDictionary, - &pixelBufferPool - ) - - try encoder?.configureCompressSession(H264EncoderConfig( - width: width, - height: height, - isRealTime: true, - expectedFrameRate: config.fps, - averageBitRate: config.bitrate, - quality: config.encoderQuality - )) - - encoder?.naluHandling = { [weak self] data in - guard let self else { return } - tcpServer?.dataHandler?(data) - } - - // Initialize TCP server - tcpServer = TCPServer() - tcpServer?.onClientConnected = { [weak self] in - guard let self = self else { - print("[H264Stream] onClientConnected: self is nil") - return - } - - print("[H264Stream] Client connected! Sending cached data...") - - guard let handler = self.tcpServer?.dataHandler else { - print("[H264Stream] ERROR: dataHandler is nil!") - return - } - - // Send cached data immediately when client connects - let sps = self.cachedSPS - let pps = self.cachedPPS - let idr = self.cachedIDR - - if let sps = sps { - print("[H264Stream] Sending cached SPS (\(sps.count) bytes)") - handler(sps) - } else { - print("[H264Stream] WARNING: No cached SPS!") - } - - if let pps = pps { - print("[H264Stream] Sending cached PPS (\(pps.count) bytes)") - handler(pps) - } else { - print("[H264Stream] WARNING: No cached PPS!") - } - - if let idr = idr { - print("[H264Stream] Sending cached IDR (\(idr.count) bytes)") - handler(idr) - } else { - print("[H264Stream] WARNING: No cached IDR!") - } - } - try tcpServer?.start(port: config.port) - - // Start capture loop - isRunning = true - frameNumber = 0 - streamTask = Task { [weak self] in - await self?.captureLoop() - } - - logger.info("H264 stream started on port \(config.port)") - } - - /// Stops the stream. - func stop() { - guard isRunning else { return } - - logger.info("Stopping H264 stream") - - isRunning = false - streamTask?.cancel() - streamTask = nil - - encoder?.invalidateCompressionSession() - encoder = nil - - tcpServer?.stop() - tcpServer = nil - - ciContext = nil - pixelBufferPool = nil - cachedSPS = nil - cachedPPS = nil - cachedIDR = nil - } - - /// Updates encoder settings dynamically. - func updateConfig(bitrate: Int? = nil, fps: Int? = nil) { - if let bitrate = bitrate { - config.bitrate = bitrate - } - if let fps = fps { - config.fps = fps - } - - try? encoder?.updateEncoderSettings( - newBitrate: config.bitrate, - newFrameRate: fps - ) - } - - // MARK: - Private - - private func captureLoop() async { - let frameInterval = UInt64(1_000_000_000 / max(1, config.fps)) - - while !Task.isCancelled && isRunning { - let startTime = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - - await captureAndEncodeFrame() - - // Maintain frame rate - let elapsed = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - startTime - if elapsed < frameInterval { - try? await Task.sleep(nanoseconds: frameInterval - elapsed) - } - } - } - - private func captureAndEncodeFrame() async { - let t0 = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - - // 1. Capture stays on MainActor - guard let uiImage = try? FBScreenshot.captureUIImage( - withQuality: config.screenshotQuality, - timeout: 0.5 - ) else { - logger.error("UIImage capture failed") - return - } - - let t1 = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - let captureMs = Double(t1 - t0) / 1_000_000 - - // 2. Heavy work off-main - let config = self.config - let ciContext = self.ciContext - let encoder = self.encoder - let pixelBufferPool = self.pixelBufferPool - let frameNumber = self.frameNumber - - Task.detached { [logger] in - let t2 = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - - guard let scaledImage = UIImageScaler.scaleImage(uiImage, scaleFactor: config.scale), - let pixelBuffer = await self.uiImageToPixelBuffer( - from: scaledImage, - size: scaledImage.size, - ) else { - logger.warning("UIImage to pixel buffer conversion failed") - return - } - - let t3 = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - let convertMs = Double(t3 - t2) / 1_000_000 - - guard let ciContext, let encoder else { return } - - let timestamp = CMTime(value: frameNumber, timescale: Int32(config.fps)) - - encoder.encode( - imageBuffer: pixelBuffer, - timestamp: timestamp, - context: ciContext, - orientation: .up - ) - - let t4 = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - let encodeSubmitMs = Double(t4 - t3) / 1_000_000 - - if frameNumber % 30 == 0 { - logger.info( - "Frame \(frameNumber): capture=\(captureMs)ms, convert=\(convertMs)ms, submit=\(encodeSubmitMs)ms" - ) - } - } - - // 3. Only frameNumber mutation stays on MainActor - self.frameNumber += 1 - } - - - func uiImageToPixelBuffer( - from image: UIImage, - size: CGSize, - ) -> CVPixelBuffer? { - - guard let cgImage = image.cgImage else { return nil } - - var pixelBuffer: CVPixelBuffer? - - // Reuse from pool if available (much faster!) - if let pool = pixelBufferPool { - CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixelBuffer) - } else { - let attrs: CFDictionary = [ - kCVPixelBufferCGImageCompatibilityKey: kCFBooleanTrue!, - kCVPixelBufferCGBitmapContextCompatibilityKey: kCFBooleanTrue!, - kCVPixelBufferIOSurfacePropertiesKey: [:] as CFDictionary - ] as CFDictionary - - CVPixelBufferCreate( - kCFAllocatorDefault, - Int(size.width), - Int(size.height), - kCVPixelFormatType_32BGRA, // Native iOS format - attrs, - &pixelBuffer - ) - } - - guard let buffer = pixelBuffer else { return nil } - - CVPixelBufferLockBaseAddress(buffer, []) - defer { CVPixelBufferUnlockBaseAddress(buffer, []) } - - guard let data = CVPixelBufferGetBaseAddress(buffer) else { return nil } - - let rgbColorSpace = CGColorSpaceCreateDeviceRGB() - - let context = CGContext( - data: data, - width: Int(size.width), - height: Int(size.height), - bitsPerComponent: 8, - bytesPerRow: CVPixelBufferGetBytesPerRow(buffer), - space: rgbColorSpace, - bitmapInfo: CGImageAlphaInfo.noneSkipFirst.rawValue | - CGBitmapInfo.byteOrder32Little.rawValue - ) - - context?.draw(cgImage, in: CGRect(origin: .zero, size: size)) - - return buffer - } - - - private func getScreenSize() -> CGSize { - let screen = UIScreen.main - return CGSize( - width: screen.bounds.width * screen.scale, - height: screen.bounds.height * screen.scale - ) - } -} diff --git a/DeviceKitTests/H264Stream/UIImageScaler.swift b/DeviceKitTests/H264Stream/UIImageScaler.swift deleted file mode 100644 index ee31923..0000000 --- a/DeviceKitTests/H264Stream/UIImageScaler.swift +++ /dev/null @@ -1,60 +0,0 @@ -import CoreGraphics -import UIKit - -/// Utility for scaling UIImages using Core Graphics. -enum UIImageScaler { - - /// Scales a UIImage by a given factor. - /// - /// - Parameters: - /// - image: Original UIImage. - /// - scaleFactor: Scale factor (0.0–1.0). 0.5 = 50% of original size. - /// - interpolation: Quality used when scaling (default: .medium). - /// - Returns: A new scaled UIImage, or nil if scaling failed. - static func scaleImage( - _ image: UIImage, - scaleFactor: CGFloat, - interpolation: CGInterpolationQuality = .medium - ) -> UIImage? { - - guard scaleFactor > 0 && scaleFactor < 1 else { - return image // No scaling needed - } - - guard let cgImage = image.cgImage else { - return nil - } - - let originalWidth = CGFloat(cgImage.width) - let originalHeight = CGFloat(cgImage.height) - - let newWidth = Int(originalWidth * scaleFactor) - let newHeight = Int(originalHeight * scaleFactor) - - guard newWidth > 0 && newHeight > 0 else { - return nil - } - - guard let context = CGContext( - data: nil, - width: newWidth, - height: newHeight, - bitsPerComponent: 8, - bytesPerRow: 0, - space: CGColorSpaceCreateDeviceRGB(), - bitmapInfo: CGImageAlphaInfo.premultipliedLast.rawValue - ) else { - return nil - } - - context.interpolationQuality = interpolation - - context.draw(cgImage, in: CGRect(x: 0, y: 0, width: newWidth, height: newHeight)) - - guard let scaledCGImage = context.makeImage() else { - return nil - } - - return UIImage(cgImage: scaledCGImage, scale: image.scale, orientation: image.imageOrientation) - } -} diff --git a/DeviceKitTests/Streamer/H264/H264.swift b/DeviceKitTests/Streamer/H264/H264.swift deleted file mode 100644 index 8f0a874..0000000 --- a/DeviceKitTests/Streamer/H264/H264.swift +++ /dev/null @@ -1,154 +0,0 @@ -import FlyingFox -import FlyingSocks -import Foundation -import os -import CoreImage -import CoreMedia -import H264Codec - -private enum H264Constants { - static let defaultFPS: Int = 30 - static let maxFPS: Int = 60 - static let defaultBitrate: Int = 4_000_000 - static let minBitrate: Int = 100_000 - static let maxBitrate: Int = 10_000_000 - static let defaultQuality: Int = 60 - static let defaultScale: Int = 50 - static let minScale: Int = 10 - static let maxScale: Int = 100 -} - -struct H264HTTPStreamConfig: Sendable { - let fps: Int - let bitrate: Int - let quality: Float - let scale: Float - let frameInterval: UInt64 - - init(fps: Int, bitrate: Int, quality: Float, scale: Float) { - self.fps = fps - self.bitrate = bitrate - self.quality = quality - self.scale = scale - self.frameInterval = UInt64(1_000_000_000 / max(1, fps)) - } -} - -@MainActor -struct H264HTTPHandler: HTTPHandler { - - private let logger = Logger( - subsystem: Bundle.main.bundleIdentifier ?? "devicekit-ios", - category: "H264Stream" - ) - - func handleRequest(_ request: HTTPRequest) async throws -> HTTPResponse { - let fps = request.queryInt(name: "fps", default: H264Constants.defaultFPS, min: 1, max: H264Constants.maxFPS) - let bitrate = request.queryInt(name: "bitrate", default: H264Constants.defaultBitrate, min: H264Constants.minBitrate, max: H264Constants.maxBitrate) - let quality = Float(request.queryInt(name: "quality", default: H264Constants.defaultQuality, min: 1, max: 100)) / 100.0 - let scalePercent = request.queryInt(name: "scale", default: H264Constants.defaultScale, min: H264Constants.minScale, max: H264Constants.maxScale) - let scale = Float(scalePercent) / 100.0 - - logger.info("Starting H264 stream: scale=\(scalePercent)% @ \(fps)fps, \(bitrate/1_000_000)Mbps") - - let config = H264HTTPStreamConfig(fps: fps, bitrate: bitrate, quality: quality, scale: scale) - let stream = H264ByteStream(config: config) - let bodySequence = HTTPBodySequence(from: stream) - - var headers: [HTTPHeader: String] = [:] - headers[.contentType] = "video/h264" - headers[HTTPHeader("Server")] = "DeviceKit-iOS" - headers[HTTPHeader("Connection")] = "close" - headers[HTTPHeader("Cache-Control")] = "no-cache, no-store, must-revalidate" - - return HTTPResponse(statusCode: .ok, headers: headers, body: bodySequence) - } - -} - -struct H264ByteStream: AsyncBufferedSequence, Sendable { - typealias Element = UInt8 - - let config: H264HTTPStreamConfig - - func makeAsyncIterator() -> H264ByteIterator { - H264ByteIterator(config: config) - } -} - -final class H264ByteIterator: AsyncBufferedIteratorProtocol, @unchecked Sendable { - typealias Element = UInt8 - typealias Buffer = [UInt8] - - private let config: H264HTTPStreamConfig - private let frameProducer: H264FrameProducer - private var naluStream: AsyncStream? - private var naluIterator: AsyncStream.Iterator? - private var captureTask: Task? - private var isCancelled = false - - private let logger = Logger( - subsystem: Bundle.main.bundleIdentifier ?? "devicekit-ios", - category: "H264Iterator" - ) - - init(config: H264HTTPStreamConfig) { - self.config = config - self.frameProducer = H264FrameProducer() - - let stream = frameProducer.makeNALUnitStream() - self.naluStream = stream - self.naluIterator = stream.makeAsyncIterator() - - startCaptureLoop() - } - - deinit { - captureTask?.cancel() - frameProducer.invalidateEncoder() - } - - private func startCaptureLoop() { - captureTask = Task { [weak self] in - while !Task.isCancelled { - guard let self = self else { return } - do { - try await self.captureFrame() - } catch { - self.logger.error("Capture error: \(error.localizedDescription)") - break - } - } - } - } - - @MainActor - private func captureFrame() async throws { - try await frameProducer.captureAndEncodeFrame( - fps: config.fps, - bitrate: config.bitrate, - quality: config.quality, - scale: config.scale, - frameInterval: config.frameInterval - ) - } - - func next() async throws -> UInt8? { - guard !isCancelled, !Task.isCancelled else { return nil } - let buffer = try await nextBuffer(suggested: 1) - return buffer?.first - } - - func nextBuffer(suggested count: Int) async throws -> [UInt8]? { - guard !isCancelled, !Task.isCancelled else { - return nil - } - - guard let data = await naluIterator?.next() else { - isCancelled = true - return nil - } - - return Array(data) - } -} diff --git a/DeviceKitTests/Streamer/H264/H264FrameProducer.swift b/DeviceKitTests/Streamer/H264/H264FrameProducer.swift deleted file mode 100644 index 71b422c..0000000 --- a/DeviceKitTests/Streamer/H264/H264FrameProducer.swift +++ /dev/null @@ -1,157 +0,0 @@ -import CoreImage -import CoreMedia -import H264Codec -import os - -enum H264Error: Error, LocalizedError { - case captureFailed - case conversionFailed - case encoderNotConfigured - - var errorDescription: String? { - switch self { - case .captureFailed: return "Screenshot capture failed" - case .conversionFailed: return "Pixel buffer conversion failed" - case .encoderNotConfigured: return "Encoder not configured" - } - } -} - -final class H264FrameProducer: @unchecked Sendable { - private let captureTimeout: TimeInterval = 0.5 - - private var encoder: H264Encoder? - private var ciContext: CIContext? - private var pixelBufferPool: CVPixelBufferPool? - private var targetSize: CGSize? - private var isConfigured = false - private var frameCount: UInt64 = 0 - - private var continuation: AsyncStream.Continuation? - - private let logger = Logger( - subsystem: Bundle.main.bundleIdentifier ?? "devicekit-ios", - category: "H264FrameProducer" - ) - - init() {} - - func makeNALUnitStream() -> AsyncStream { - AsyncStream { continuation in - self.continuation = continuation - - continuation.onTermination = { [weak self] _ in - self?.invalidateEncoder() - } - } - } - - func invalidateEncoder() { - encoder?.invalidateCompressionSession() - continuation?.finish() - continuation = nil - } - - @MainActor - func captureAndEncodeFrame( - fps: Int, - bitrate: Int, - quality: Float, - scale: Float, - frameInterval: UInt64 - ) async throws { - let frameStart = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - - guard let uiImage = try? FBScreenshot.captureUIImage( - withQuality: 0.9, - timeout: captureTimeout - ), let cgImage = uiImage.cgImage else { - throw H264Error.captureFailed - } - - if !isConfigured { - try configureEncoder( - for: cgImage, - fps: fps, - bitrate: bitrate, - quality: quality, - scale: scale - ) - } - - guard let ciContext = ciContext, - let targetSize = targetSize, - let encoder = encoder else { - throw H264Error.encoderNotConfigured - } - - guard let pixelBuffer = cgImage.toPixelBuffer( - context: ciContext, - targetSize: targetSize, - pool: pixelBufferPool - ) else { - throw H264Error.conversionFailed - } - - let timestamp = CMTime( - value: CMTimeValue(frameCount), - timescale: CMTimeScale(fps) - ) - encoder.encode(pixelBuffer: pixelBuffer, timestamp: timestamp) - frameCount += 1 - - let elapsed = clock_gettime_nsec_np(CLOCK_MONOTONIC_RAW) - frameStart - if elapsed < frameInterval { - try await Task.sleep(nanoseconds: frameInterval - elapsed) - } - } - - private func configureEncoder( - for cgImage: CGImage, - fps: Int, - bitrate: Int, - quality: Float, - scale: Float - ) throws { - let originalWidth = CGFloat(cgImage.width) - let originalHeight = CGFloat(cgImage.height) - - var scaledWidth = Int(originalWidth * CGFloat(scale)) - var scaledHeight = Int(originalHeight * CGFloat(scale)) - scaledWidth = scaledWidth - (scaledWidth % 2) - scaledHeight = scaledHeight - (scaledHeight % 2) - scaledWidth = max(64, scaledWidth) - scaledHeight = max(64, scaledHeight) - - targetSize = CGSize(width: scaledWidth, height: scaledHeight) - - logger.info("Encoder: \(scaledWidth)x\(scaledHeight) @ \(fps)fps, \(bitrate/1_000_000)Mbps") - - ciContext = CIContext(options: [ - .useSoftwareRenderer: false, - .highQualityDownsample: false - ]) - - pixelBufferPool = CGImage.createPixelBufferPool( - size: targetSize!, - minimumBufferCount: 3 - ) - - let enc = H264Encoder() - try enc.configureCompressSession(H264EncoderConfig( - width: Int32(scaledWidth), - height: Int32(scaledHeight), - isRealTime: true, - expectedFrameRate: fps, - averageBitRate: bitrate, - quality: quality - )) - - enc.naluHandling = { [weak self] data in - self?.continuation?.yield(data) - } - - encoder = enc - isConfigured = true - } -} diff --git a/DeviceKitTests/XCTestServer.swift b/DeviceKitTests/XCTestServer.swift index 0597b6c..b896a85 100644 --- a/DeviceKitTests/XCTestServer.swift +++ b/DeviceKitTests/XCTestServer.swift @@ -131,11 +131,7 @@ final class XCTestServer { let mjpegHandler = MJPEGHTTPHandler() await server.appendRoute("GET /mjpeg", to: mjpegHandler) - // H264 streaming endpoint - let h264Handler = H264HTTPHandler() - await server.appendRoute("GET /h264", to: h264Handler) - - logger.info("Server is ready (WebSocket: ws://\(self.localhost):\(port)/ws, HTTP: POST http://\(self.localhost):\(port)/rpc, MJPEG: http://\(self.localhost):\(port)/mjpeg, H264: http://\(self.localhost):\(port)/h264)") + logger.info("Server is ready (WebSocket: ws://\(self.localhost):\(port)/ws, HTTP: POST http://\(self.localhost):\(port)/rpc, MJPEG: http://\(self.localhost):\(port)/mjpeg)") try await server.run() } } diff --git a/LICENSE b/LICENSE index 261eeb9..3212f23 100644 --- a/LICENSE +++ b/LICENSE @@ -1,201 +1,45 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ +Functional Source License, Version 1.1, ALv2 Future License - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION +Copyright 2025-2026 Mobile Next HQ, Inc. - 1. Definitions. +Terms and Conditions - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. +Licensor ("We") +The party offering the Software under these Terms and Conditions. - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. +The Software +The "Software" is each version of the software that we make available under these Terms and Conditions, as indicated by our inclusion of these Terms and Conditions with the Software. - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. +License Grant +Subject to your compliance with this License Grant and the Patents, Redistribution and Trademark clauses below, we hereby grant you the right to use, copy, modify, create derivative works, publicly perform, publicly display and redistribute the Software for any Permitted Purpose identified below. - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. +Permitted Purpose +A Permitted Purpose is any purpose other than a Competing Use. A Competing Use means making the Software available to others in a commercial product or service that: - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. +1. substitutes for the Software; +2. substitutes for any other product or service we offer using the Software that exists as of the date we make the Software available; or +3. offers the same or substantially similar functionality as the Software. - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. +Permitted Purposes specifically include using the Software: - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). +1. for your internal use and access; +2. for non-commercial education; +3. for non-commercial research; and +4. in connection with professional services that you provide to a licensee using the Software in accordance with these Terms and Conditions. - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. +Patents +To the extent your use for a Permitted Purpose would necessarily infringe our patents, the license grant above includes a license under our patents. If you make a claim against any party that the Software infringes or contributes to the infringement of any patent, then your patent license to the Software ends immediately. - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." +Redistribution +The Terms and Conditions apply to all copies, modifications and derivatives of the Software. - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. +If you redistribute any copies, modifications or derivatives of the Software, you must include a copy of or a link to these Terms and Conditions and not remove any copyright notices provided in or with the Software. - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. +Disclaimer +THE SOFTWARE IS PROVIDED "AS IS" AND WITHOUT WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION WARRANTIES OF FITNESS FOR A PARTICULAR PURPOSE, MERCHANTABILITY, TITLE OR NON-INFRINGEMENT. - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. +Trademarks +Except for displaying the License Details and identifying us as the origin of the Software, you have no right under these Terms and Conditions to use our trademarks, trade names, service marks or product names. - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Grant of Future License +An additional Apache License, Version 2.0 becomes effective on the second anniversary of the Software's initial availability. After that date, users may alternatively use the Software under the Apache License, Version 2.0. diff --git a/README.md b/README.md index 311ece0..52c391e 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Swift](https://img.shields.io/badge/Swift-5.9-orange.svg?style=flat-square)](https://swift.org) [![Platform](https://img.shields.io/badge/Platform-iOS%2016.0+-blue.svg?style=flat-square)](https://developer.apple.com/ios/) -[![License](https://img.shields.io/badge/License-Apache%202.0-lightgrey.svg?style=flat-square)](LICENSE) +[![License](https://img.shields.io/badge/License-FSL--1.1--Apache--2.0-lightgrey.svg?style=flat-square)](LICENSE) Control any iOS device or simulator over a simple JSON-RPC API. Tap, swipe, stream video, inspect the UI, from any language, over localhost. @@ -229,4 +229,4 @@ Want to contribute? PRs are appreciated. Check open issues for good first tasks. ## License -DeviceKit iOS is released under the Apache 2.0 License. See [LICENSE](LICENSE) for details. +DeviceKit iOS is released under the [Functional Source License 1.1, Apache 2.0 Future License](LICENSE). diff --git a/devicekit-ios.xcodeproj/project.pbxproj b/devicekit-ios.xcodeproj/project.pbxproj index 5012f56..9f7f102 100644 --- a/devicekit-ios.xcodeproj/project.pbxproj +++ b/devicekit-ios.xcodeproj/project.pbxproj @@ -48,7 +48,6 @@ 781480372F27CEF000E61E3B /* Screenshot.swift in Sources */ = {isa = PBXBuildFile; fileRef = 781480362F27CEF000E61E3B /* Screenshot.swift */; }; 781480752F27D2BB00E61E3B /* URL.swift in Sources */ = {isa = PBXBuildFile; fileRef = 781480742F27D2B900E61E3B /* URL.swift */; }; 781480FF2F27DEE600E61E3B /* IOGesture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 781480FE2F27DEE600E61E3B /* IOGesture.swift */; }; - 7844BCC12F28128A00F7E003 /* OpusCodec in Frameworks */ = {isa = PBXBuildFile; productRef = 7844BCC02F28128A00F7E003 /* OpusCodec */; }; 785281492F2BEC40002A7531 /* FBScreenshot.h in Headers */ = {isa = PBXBuildFile; fileRef = 785281472F2BEC40002A7531 /* FBScreenshot.h */; }; 7852814A2F2BEC40002A7531 /* FBScreenshot.m in Sources */ = {isa = PBXBuildFile; fileRef = 785281482F2BEC40002A7531 /* FBScreenshot.m */; }; 7865E4C42F423A8700D0F45F /* IOOrientationSet.swift in Sources */ = {isa = PBXBuildFile; fileRef = 7865E4C32F423A8700D0F45F /* IOOrientationSet.swift */; }; @@ -65,11 +64,7 @@ 787C48482F228D960027A012 /* JSONRPCMessageHandler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 787C48472F228D950027A012 /* JSONRPCMessageHandler.swift */; }; 787C95982F17F23500AB3BF1 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 787C95972F17F23500AB3BF1 /* Assets.xcassets */; }; 78A190B62F3A3ADA007F24E0 /* IOButton.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78A190B52F3A3ADA007F24E0 /* IOButton.swift */; }; - 78C54B352F2CF23800A9369C /* H264Codec in Frameworks */ = {isa = PBXBuildFile; productRef = 78C54B342F2CF23800A9369C /* H264Codec */; }; 78EA9F912F36611B008B17D3 /* AppsTerminate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78EA9F902F36611B008B17D3 /* AppsTerminate.swift */; }; - 78F689082F17EE190025C8AC /* ReplayKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 78F689072F17EE190025C8AC /* ReplayKit.framework */; }; - 78F6891E2F17EE190025C8AC /* BroadcastUploadExtension.appex in Embed Foundation Extensions */ = {isa = PBXBuildFile; fileRef = 78F689062F17EE190025C8AC /* BroadcastUploadExtension.appex */; settings = {ATTRIBUTES = (RemoveHeadersOnCopy, ); }; }; - 78F689312F17EE5C0025C8AC /* H264Codec in Frameworks */ = {isa = PBXBuildFile; productRef = 78F689302F17EE5C0025C8AC /* H264Codec */; }; 78F689392F17EF1B0025C8AC /* devicekit_iosApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 78F689372F17EF1B0025C8AC /* devicekit_iosApp.swift */; }; 94826F4429DB082100795E9B /* SystemPermissionManager.swift in Sources */ = {isa = PBXBuildFile; fileRef = 94826F4329DB082100795E9B /* SystemPermissionManager.swift */; }; 9811C9092C49751D00DDACA0 /* OrientationGeometry.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9811C9082C49751D00DDACA0 /* OrientationGeometry.swift */; }; @@ -84,29 +79,8 @@ remoteGlobalIDString = 52049BD22935039F00807AA3; remoteInfo = "maestro-driver-ios"; }; - 78F6891C2F17EE190025C8AC /* PBXContainerItemProxy */ = { - isa = PBXContainerItemProxy; - containerPortal = 52049BCB2935039F00807AA3 /* Project object */; - proxyType = 1; - remoteGlobalIDString = 78F689052F17EE190025C8AC; - remoteInfo = BroadcastUploadExtension; - }; /* End PBXContainerItemProxy section */ -/* Begin PBXCopyFilesBuildPhase section */ - 78F689272F17EE190025C8AC /* Embed Foundation Extensions */ = { - isa = PBXCopyFilesBuildPhase; - buildActionMask = 2147483647; - dstPath = ""; - dstSubfolderSpec = 13; - files = ( - 78F6891E2F17EE190025C8AC /* BroadcastUploadExtension.appex in Embed Foundation Extensions */, - ); - name = "Embed Foundation Extensions"; - runOnlyForDeploymentPostprocessing = 0; - }; -/* End PBXCopyFilesBuildPhase section */ - /* Begin PBXFileReference section */ 52049BD32935039F00807AA3 /* devicekit-ios.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "devicekit-ios.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 52049BF3293503A200807AA3 /* devicekit-iosUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "devicekit-iosUITests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -173,8 +147,6 @@ 787C95972F17F23500AB3BF1 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; 78A190B52F3A3ADA007F24E0 /* IOButton.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = IOButton.swift; sourceTree = ""; }; 78EA9F902F36611B008B17D3 /* AppsTerminate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppsTerminate.swift; sourceTree = ""; }; - 78F689062F17EE190025C8AC /* BroadcastUploadExtension.appex */ = {isa = PBXFileReference; explicitFileType = "wrapper.app-extension"; includeInIndex = 0; path = BroadcastUploadExtension.appex; sourceTree = BUILT_PRODUCTS_DIR; }; - 78F689072F17EE190025C8AC /* ReplayKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = ReplayKit.framework; path = System/Library/Frameworks/ReplayKit.framework; sourceTree = SDKROOT; }; 78F689132F17EE190025C8AC /* UIKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = UIKit.framework; path = System/Library/Frameworks/UIKit.framework; sourceTree = SDKROOT; }; 78F689372F17EF1B0025C8AC /* devicekit_iosApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = devicekit_iosApp.swift; sourceTree = ""; }; 94826F4329DB082100795E9B /* SystemPermissionManager.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SystemPermissionManager.swift; sourceTree = ""; }; @@ -182,20 +154,9 @@ AAAA00012F82000100000001 /* Assets.xcassets */ = {isa = PBXFileReference; lastKnownFileType = folder.assetcatalog; path = Assets.xcassets; sourceTree = ""; }; /* End PBXFileReference section */ -/* Begin PBXFileSystemSynchronizedBuildFileExceptionSet section */ - 78F6891F2F17EE190025C8AC /* PBXFileSystemSynchronizedBuildFileExceptionSet */ = { - isa = PBXFileSystemSynchronizedBuildFileExceptionSet; - membershipExceptions = ( - Info.plist, - ); - target = 78F689052F17EE190025C8AC /* BroadcastUploadExtension */; - }; -/* End PBXFileSystemSynchronizedBuildFileExceptionSet section */ - /* Begin PBXFileSystemSynchronizedRootGroup section */ 787C95D52F17F59E00AB3BF1 /* View */ = {isa = PBXFileSystemSynchronizedRootGroup; explicitFileTypes = {}; explicitFolders = (); path = View; sourceTree = ""; }; 78C54BD12F2D67E300A9369C /* Streamer */ = {isa = PBXFileSystemSynchronizedRootGroup; explicitFileTypes = {}; explicitFolders = (); path = Streamer; sourceTree = ""; }; - 78F689092F17EE190025C8AC /* BroadcastUploadExtension */ = {isa = PBXFileSystemSynchronizedRootGroup; exceptions = (78F6891F2F17EE190025C8AC /* PBXFileSystemSynchronizedBuildFileExceptionSet */, ); explicitFileTypes = {}; explicitFolders = (); path = BroadcastUploadExtension; sourceTree = ""; }; /* End PBXFileSystemSynchronizedRootGroup section */ /* Begin PBXFrameworksBuildPhase section */ @@ -212,17 +173,6 @@ files = ( 612DE40229801F71003C2BE0 /* XCTest.framework in Frameworks */, 5279BFD62935ECE20056C609 /* FlyingFox in Frameworks */, - 78C54B352F2CF23800A9369C /* H264Codec in Frameworks */, - ); - runOnlyForDeploymentPostprocessing = 0; - }; - 78F689032F17EE190025C8AC /* Frameworks */ = { - isa = PBXFrameworksBuildPhase; - buildActionMask = 2147483647; - files = ( - 78F689082F17EE190025C8AC /* ReplayKit.framework in Frameworks */, - 7844BCC12F28128A00F7E003 /* OpusCodec in Frameworks */, - 78F689312F17EE5C0025C8AC /* H264Codec in Frameworks */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -234,7 +184,6 @@ children = ( 52049BD52935039F00807AA3 /* DeviceKit */, 52049BF6293503A200807AA3 /* DeviceKitTests */, - 78F689092F17EE190025C8AC /* BroadcastUploadExtension */, 52049BD42935039F00807AA3 /* Products */, 5279BFD42935ECE20056C609 /* Frameworks */, ); @@ -245,7 +194,6 @@ children = ( 52049BD32935039F00807AA3 /* devicekit-ios.app */, 52049BF3293503A200807AA3 /* devicekit-iosUITests.xctest */, - 78F689062F17EE190025C8AC /* BroadcastUploadExtension.appex */, ); name = Products; sourceTree = ""; @@ -280,7 +228,6 @@ isa = PBXGroup; children = ( 612DE40129801F71003C2BE0 /* XCTest.framework */, - 78F689072F17EE190025C8AC /* ReplayKit.framework */, 78F689132F17EE190025C8AC /* UIKit.framework */, ); name = Frameworks; @@ -436,12 +383,10 @@ 52049BCF2935039F00807AA3 /* Sources */, 52049BD02935039F00807AA3 /* Frameworks */, 52049BD12935039F00807AA3 /* Resources */, - 78F689272F17EE190025C8AC /* Embed Foundation Extensions */, ); buildRules = ( ); dependencies = ( - 78F6891D2F17EE190025C8AC /* PBXTargetDependency */, ); fileSystemSynchronizedGroups = ( 787C95D52F17F59E00AB3BF1 /* View */, @@ -473,36 +418,11 @@ name = "devicekit-iosUITests"; packageProductDependencies = ( 5279BFD52935ECE20056C609 /* FlyingFox */, - 78C54B342F2CF23800A9369C /* H264Codec */, ); productName = "maestro-driver-iosUITests"; productReference = 52049BF3293503A200807AA3 /* devicekit-iosUITests.xctest */; productType = "com.apple.product-type.bundle.ui-testing"; }; - 78F689052F17EE190025C8AC /* BroadcastUploadExtension */ = { - isa = PBXNativeTarget; - buildConfigurationList = 78F689202F17EE190025C8AC /* Build configuration list for PBXNativeTarget "BroadcastUploadExtension" */; - buildPhases = ( - 78F689022F17EE190025C8AC /* Sources */, - 78F689032F17EE190025C8AC /* Frameworks */, - 78F689042F17EE190025C8AC /* Resources */, - ); - buildRules = ( - ); - dependencies = ( - ); - fileSystemSynchronizedGroups = ( - 78F689092F17EE190025C8AC /* BroadcastUploadExtension */, - ); - name = BroadcastUploadExtension; - packageProductDependencies = ( - 78F689302F17EE5C0025C8AC /* H264Codec */, - 7844BCC02F28128A00F7E003 /* OpusCodec */, - ); - productName = BroadcastUploadExtension; - productReference = 78F689062F17EE190025C8AC /* BroadcastUploadExtension.appex */; - productType = "com.apple.product-type.app-extension"; - }; /* End PBXNativeTarget section */ /* Begin PBXProject section */ @@ -521,9 +441,6 @@ LastSwiftMigration = 1620; TestTargetID = 52049BD22935039F00807AA3; }; - 78F689052F17EE190025C8AC = { - CreatedOnToolsVersion = 26.1; - }; }; }; buildConfigurationList = 52049BCE2935039F00807AA3 /* Build configuration list for PBXProject "devicekit-ios" */; @@ -537,8 +454,6 @@ mainGroup = 52049BCA2935039F00807AA3; packageReferences = ( 52049C062935E04D00807AA3 /* XCRemoteSwiftPackageReference "FlyingFox" */, - 78F689002F17EDEB0025C8AC /* XCLocalSwiftPackageReference "h264-codec" */, - 7844BCBF2F28128A00F7E003 /* XCLocalSwiftPackageReference "opus-codec" */, ); productRefGroup = 52049BD42935039F00807AA3 /* Products */; projectDirPath = ""; @@ -546,7 +461,6 @@ targets = ( 52049BD22935039F00807AA3 /* devicekit-ios */, 52049BF2293503A200807AA3 /* devicekit-iosUITests */, - 78F689052F17EE190025C8AC /* BroadcastUploadExtension */, ); }; /* End PBXProject section */ @@ -568,13 +482,6 @@ ); runOnlyForDeploymentPostprocessing = 0; }; - 78F689042F17EE190025C8AC /* Resources */ = { - isa = PBXResourcesBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - runOnlyForDeploymentPostprocessing = 0; - }; /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ @@ -632,13 +539,6 @@ ); runOnlyForDeploymentPostprocessing = 0; }; - 78F689022F17EE190025C8AC /* Sources */ = { - isa = PBXSourcesBuildPhase; - buildActionMask = 2147483647; - files = ( - ); - runOnlyForDeploymentPostprocessing = 0; - }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ @@ -647,11 +547,6 @@ target = 52049BD22935039F00807AA3 /* devicekit-ios */; targetProxy = 52049BF4293503A200807AA3 /* PBXContainerItemProxy */; }; - 78F6891D2F17EE190025C8AC /* PBXTargetDependency */ = { - isa = PBXTargetDependency; - target = 78F689052F17EE190025C8AC /* BroadcastUploadExtension */; - targetProxy = 78F6891C2F17EE190025C8AC /* PBXContainerItemProxy */; - }; /* End PBXTargetDependency section */ /* Begin XCBuildConfiguration section */ @@ -901,107 +796,6 @@ }; name = Release; }; - 78F689212F17EE190025C8AC /* Debug */ = { - isa = XCBuildConfiguration; - buildSettings = { - ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; - CODE_SIGN_IDENTITY = "Apple Development"; - CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = UWTTLVVSRY; - ENABLE_USER_SCRIPT_SANDBOXING = YES; - GCC_C_LANGUAGE_STANDARD = gnu17; - GENERATE_INFOPLIST_FILE = YES; - HEADER_SEARCH_PATHS = ( - "$(inherited)", - "$(SRCROOT)/ThirdParty/opus/include", - "$(SRCROOT)/ThirdParty/opus/src", - "$(SRCROOT)/ThirdParty/opus/celt", - "$(SRCROOT)/ThirdParty/opus/silk", - "$(SRCROOT)/ThirdParty/opus/silk/float", - ); - INFOPLIST_FILE = BroadcastUploadExtension/Info.plist; - INFOPLIST_KEY_CFBundleDisplayName = BroadcastUploadExtension; - INFOPLIST_KEY_NSHumanReadableCopyright = ""; - IPHONEOS_DEPLOYMENT_TARGET = 16.0; - LD_RUNPATH_SEARCH_PATHS = ( - "$(inherited)", - "@executable_path/Frameworks", - "@executable_path/../../Frameworks", - ); - LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MARKETING_VERSION = 1.0; - OTHER_CFLAGS = ( - "$(inherited)", - "-DOPUS_BUILD", - "-DVAR_ARRAYS", - ); - PRODUCT_BUNDLE_IDENTIFIER = "com.mobilenext.devicekit-ios.BroadcastUploadExtension"; - PRODUCT_NAME = "$(TARGET_NAME)"; - PROVISIONING_PROFILE_SPECIFIER = ""; - SKIP_INSTALL = YES; - STRING_CATALOG_GENERATE_SYMBOLS = YES; - SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; - SWIFT_APPROACHABLE_CONCURRENCY = YES; - SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_INCLUDE_PATHS = "$(inherited) $(SRCROOT)/ThirdParty/opus/include $(SRCROOT)/ThirdParty/opus/src $(SRCROOT)/ThirdParty/opus/celt $(SRCROOT)/ThirdParty/opus/silk $(SRCROOT)/ThirdParty/opus/silk/float"; - SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.9; - TARGETED_DEVICE_FAMILY = "1,2"; - }; - name = Debug; - }; - 78F689222F17EE190025C8AC /* Release */ = { - isa = XCBuildConfiguration; - buildSettings = { - ASSETCATALOG_COMPILER_GENERATE_SWIFT_ASSET_SYMBOL_EXTENSIONS = YES; - CLANG_CXX_LANGUAGE_STANDARD = "gnu++20"; - CODE_SIGN_IDENTITY = "Apple Development"; - CODE_SIGN_STYLE = Automatic; - CURRENT_PROJECT_VERSION = 1; - DEVELOPMENT_TEAM = UWTTLVVSRY; - ENABLE_USER_SCRIPT_SANDBOXING = YES; - GCC_C_LANGUAGE_STANDARD = gnu17; - GENERATE_INFOPLIST_FILE = YES; - HEADER_SEARCH_PATHS = ( - "$(inherited)", - "$(SRCROOT)/ThirdParty/opus/include", - "$(SRCROOT)/ThirdParty/opus/src", - "$(SRCROOT)/ThirdParty/opus/celt", - "$(SRCROOT)/ThirdParty/opus/silk", - "$(SRCROOT)/ThirdParty/opus/silk/float", - ); - INFOPLIST_FILE = BroadcastUploadExtension/Info.plist; - INFOPLIST_KEY_CFBundleDisplayName = BroadcastUploadExtension; - INFOPLIST_KEY_NSHumanReadableCopyright = ""; - IPHONEOS_DEPLOYMENT_TARGET = 16.0; - LD_RUNPATH_SEARCH_PATHS = ( - "$(inherited)", - "@executable_path/Frameworks", - "@executable_path/../../Frameworks", - ); - LOCALIZATION_PREFERS_STRING_CATALOGS = YES; - MARKETING_VERSION = 1.0; - OTHER_CFLAGS = ( - "$(inherited)", - "-DOPUS_BUILD", - "-DVAR_ARRAYS", - ); - PRODUCT_BUNDLE_IDENTIFIER = "com.mobilenext.devicekit-ios.BroadcastUploadExtension"; - PRODUCT_NAME = "$(TARGET_NAME)"; - PROVISIONING_PROFILE_SPECIFIER = ""; - SKIP_INSTALL = YES; - STRING_CATALOG_GENERATE_SYMBOLS = YES; - SWIFT_APPROACHABLE_CONCURRENCY = YES; - SWIFT_EMIT_LOC_STRINGS = YES; - SWIFT_INCLUDE_PATHS = "$(inherited) $(SRCROOT)/ThirdParty/opus/include $(SRCROOT)/ThirdParty/opus/src $(SRCROOT)/ThirdParty/opus/celt $(SRCROOT)/ThirdParty/opus/silk $(SRCROOT)/ThirdParty/opus/silk/float"; - SWIFT_UPCOMING_FEATURE_MEMBER_IMPORT_VISIBILITY = YES; - SWIFT_VERSION = 5.9; - TARGETED_DEVICE_FAMILY = "1,2"; - }; - name = Release; - }; /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ @@ -1032,28 +826,8 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 78F689202F17EE190025C8AC /* Build configuration list for PBXNativeTarget "BroadcastUploadExtension" */ = { - isa = XCConfigurationList; - buildConfigurations = ( - 78F689212F17EE190025C8AC /* Debug */, - 78F689222F17EE190025C8AC /* Release */, - ); - defaultConfigurationIsVisible = 0; - defaultConfigurationName = Release; - }; /* End XCConfigurationList section */ -/* Begin XCLocalSwiftPackageReference section */ - 7844BCBF2F28128A00F7E003 /* XCLocalSwiftPackageReference "opus-codec" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = "opus-codec"; - }; - 78F689002F17EDEB0025C8AC /* XCLocalSwiftPackageReference "h264-codec" */ = { - isa = XCLocalSwiftPackageReference; - relativePath = "h264-codec"; - }; -/* End XCLocalSwiftPackageReference section */ - /* Begin XCRemoteSwiftPackageReference section */ 52049C062935E04D00807AA3 /* XCRemoteSwiftPackageReference "FlyingFox" */ = { isa = XCRemoteSwiftPackageReference; @@ -1071,20 +845,6 @@ package = 52049C062935E04D00807AA3 /* XCRemoteSwiftPackageReference "FlyingFox" */; productName = FlyingFox; }; - 7844BCC02F28128A00F7E003 /* OpusCodec */ = { - isa = XCSwiftPackageProductDependency; - productName = OpusCodec; - }; - 78C54B342F2CF23800A9369C /* H264Codec */ = { - isa = XCSwiftPackageProductDependency; - package = 78F689002F17EDEB0025C8AC /* XCLocalSwiftPackageReference "h264-codec" */; - productName = H264Codec; - }; - 78F689302F17EE5C0025C8AC /* H264Codec */ = { - isa = XCSwiftPackageProductDependency; - package = 78F689002F17EDEB0025C8AC /* XCLocalSwiftPackageReference "h264-codec" */; - productName = H264Codec; - }; /* End XCSwiftPackageProductDependency section */ }; rootObject = 52049BCB2935039F00807AA3 /* Project object */; diff --git a/h264-codec/.gitignore b/h264-codec/.gitignore deleted file mode 100644 index 0023a53..0000000 --- a/h264-codec/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.DS_Store -/.build -/Packages -xcuserdata/ -DerivedData/ -.swiftpm/configuration/registries.json -.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata -.netrc diff --git a/h264-codec/Package.swift b/h264-codec/Package.swift deleted file mode 100644 index 2a64163..0000000 --- a/h264-codec/Package.swift +++ /dev/null @@ -1,19 +0,0 @@ -// swift-tools-version: 5.9 - -import PackageDescription - -let package = Package( - name: "H264Codec", - products: [ - .library( - name: "H264Codec", - targets: ["H264Codec"]) - ], - targets: [ - .target( - name: "H264Codec"), - .testTarget( - name: "H264CodecTests", - dependencies: ["H264Codec"]) - ] -) diff --git a/h264-codec/Sources/h264-codec/CGImage+Extension.swift b/h264-codec/Sources/h264-codec/CGImage+Extension.swift deleted file mode 100644 index 9b3e7b8..0000000 --- a/h264-codec/Sources/h264-codec/CGImage+Extension.swift +++ /dev/null @@ -1,86 +0,0 @@ -import CoreGraphics -import CoreImage -import CoreVideo - -extension CGImage { - public func toPixelBuffer( - context: CIContext, - targetSize: CGSize? = nil, - pixelFormat: OSType = kCVPixelFormatType_32BGRA, - pool: CVPixelBufferPool? = nil - ) -> CVPixelBuffer? { - var ciImage = CIImage(cgImage: self) - - let outputSize = targetSize ?? CGSize(width: width, height: height) - - let scaleX = outputSize.width / ciImage.extent.width - let scaleY = outputSize.height / ciImage.extent.height - if abs(scaleX - 1.0) > 0.001 || abs(scaleY - 1.0) > 0.001 { - ciImage = ciImage.transformed(by: CGAffineTransform(scaleX: scaleX, y: scaleY)) - } - - var pixelBuffer: CVPixelBuffer? - - if let pool = pool { - let status = CVPixelBufferPoolCreatePixelBuffer(kCFAllocatorDefault, pool, &pixelBuffer) - if status != kCVReturnSuccess { - return nil - } - } else { - let attributes: [String: Any] = [ - kCVPixelBufferPixelFormatTypeKey as String: pixelFormat, - kCVPixelBufferWidthKey as String: Int(outputSize.width), - kCVPixelBufferHeightKey as String: Int(outputSize.height), - kCVPixelBufferIOSurfacePropertiesKey as String: [:], - kCVPixelBufferMetalCompatibilityKey as String: true - ] - - let status = CVPixelBufferCreate( - kCFAllocatorDefault, - Int(outputSize.width), - Int(outputSize.height), - pixelFormat, - attributes as CFDictionary, - &pixelBuffer - ) - - if status != kCVReturnSuccess { - return nil - } - } - - guard let buffer = pixelBuffer else { return nil } - - context.render(ciImage, to: buffer) - - return buffer - } - - public static func createPixelBufferPool( - size: CGSize, - pixelFormat: OSType = kCVPixelFormatType_32BGRA, - minimumBufferCount: Int = 6 - ) -> CVPixelBufferPool? { - let poolAttributes: [String: Any] = [ - kCVPixelBufferPoolMinimumBufferCountKey as String: minimumBufferCount - ] - - let pixelBufferAttributes: [String: Any] = [ - kCVPixelBufferPixelFormatTypeKey as String: pixelFormat, - kCVPixelBufferWidthKey as String: Int(size.width), - kCVPixelBufferHeightKey as String: Int(size.height), - kCVPixelBufferIOSurfacePropertiesKey as String: [:], - kCVPixelBufferMetalCompatibilityKey as String: true - ] - - var pool: CVPixelBufferPool? - let status = CVPixelBufferPoolCreate( - kCFAllocatorDefault, - poolAttributes as CFDictionary, - pixelBufferAttributes as CFDictionary, - &pool - ) - - return status == kCVReturnSuccess ? pool : nil - } -} diff --git a/h264-codec/Sources/h264-codec/CIImage+Extension.swift b/h264-codec/Sources/h264-codec/CIImage+Extension.swift deleted file mode 100644 index 2fd760b..0000000 --- a/h264-codec/Sources/h264-codec/CIImage+Extension.swift +++ /dev/null @@ -1,16 +0,0 @@ -import CoreImage - -extension CIImage { - func flip(orientation: CGImagePropertyOrientation) -> CIImage { - switch orientation { - case .up, .upMirrored, .down, .downMirrored: - return oriented(.up) - - case .left, .leftMirrored: - return oriented(.right) - - case .right, .rightMirrored: - return oriented(.left) - } - } -} diff --git a/h264-codec/Sources/h264-codec/CMSampleBuffer+Extension.swift b/h264-codec/Sources/h264-codec/CMSampleBuffer+Extension.swift deleted file mode 100644 index cc77dbf..0000000 --- a/h264-codec/Sources/h264-codec/CMSampleBuffer+Extension.swift +++ /dev/null @@ -1,13 +0,0 @@ -import CoreMedia - -extension CMSampleBuffer { - var isKeyFrame: Bool { - let attachments = CMSampleBufferGetSampleAttachmentsArray( - self, - createIfNecessary: true - ) as? [[CFString: Any]] - - let isNotKeyFrame = (attachments?.first?[kCMSampleAttachmentKey_NotSync] as? Bool) ?? false - return !isNotKeyFrame - } -} diff --git a/h264-codec/Sources/h264-codec/CVImageBuffer+Extension.swift b/h264-codec/Sources/h264-codec/CVImageBuffer+Extension.swift deleted file mode 100644 index 2f55a0f..0000000 --- a/h264-codec/Sources/h264-codec/CVImageBuffer+Extension.swift +++ /dev/null @@ -1,37 +0,0 @@ -import CoreImage -import CoreMedia - -extension CVImageBuffer { - func rotate(context: CIContext, orientation: CGImagePropertyOrientation) -> CVPixelBuffer? { - CVPixelBufferLockBaseAddress(self, .readOnly) - - let ciImage = CIImage(cvPixelBuffer: self) - let rotatedCIImage = ciImage.flip(orientation: orientation) - - if let rotatedPixelBuffer = rotatedCIImage.pixelBuffer { - CVPixelBufferUnlockBaseAddress(self, .readOnly) - return rotatedPixelBuffer - } - - var rotatedPixelBuffer: CVPixelBuffer? - CVPixelBufferCreate( - kCFAllocatorDefault, - Int(rotatedCIImage.extent.width), - Int(rotatedCIImage.extent.height), - CVPixelBufferGetPixelFormatType(self), - nil, - &rotatedPixelBuffer - ) - - guard let rotatedBuffer = rotatedPixelBuffer else { - CVPixelBufferUnlockBaseAddress(self, .readOnly) - return nil - } - - context.render(rotatedCIImage, to: rotatedBuffer) - - CVPixelBufferUnlockBaseAddress(self, .readOnly) - - return rotatedBuffer - } -} diff --git a/h264-codec/Sources/h264-codec/H264Encoder.swift b/h264-codec/Sources/h264-codec/H264Encoder.swift deleted file mode 100644 index 4b2ee04..0000000 --- a/h264-codec/Sources/h264-codec/H264Encoder.swift +++ /dev/null @@ -1,386 +0,0 @@ -import Accelerate -import CoreGraphics -import CoreImage -import CoreMedia -import CoreVideo -import VideoToolbox - -public struct H264EncoderConfig { - public let width: Int32 - public let height: Int32 - public let isRealTime: Bool - public let expectedFrameRate: Int - public let averageBitRate: Int - public let quality: Float - - public init( - width: Int32, - height: Int32, - isRealTime: Bool, - expectedFrameRate: Int, - averageBitRate: Int, - quality: Float - ) { - self.width = width - self.height = height - self.isRealTime = isRealTime - self.expectedFrameRate = expectedFrameRate - self.averageBitRate = averageBitRate - self.quality = quality - } -} - -public final class H264Encoder: NSObject { - enum ConfigurationError: Error { - case cannotCreateSession - case cannotSetProperties - case cannotPrepareToEncode - } - - private var session: VTCompressionSession? - - private static let naluStartCode = Data([UInt8](arrayLiteral: 0x00, 0x00, 0x00, 0x01)) - - // uuid for timing SEI (user data unregistered) - private static let timingUUID: [UInt8] = [ - 0x4D, 0x4F, 0x42, 0x49, 0x4C, 0x45, 0x4E, 0x58, // "MOBILENX" - 0x54, 0x49, 0x4D, 0x45, 0x43, 0x4F, 0x44, 0x45 // "TIMECODE" - ] - - public var naluHandling: ((Data) -> Void)? - - public func configureCompressSession(_ config: H264EncoderConfig) throws { - let error = VTCompressionSessionCreate( - allocator: kCFAllocatorDefault, - width: config.width, - height: config.height, - codecType: kCMVideoCodecType_H264, - encoderSpecification: nil, - imageBufferAttributes: nil, - compressedDataAllocator: kCFAllocatorDefault, - outputCallback: encodingOutputCallback, - refcon: Unmanaged.passUnretained(self).toOpaque(), - compressionSessionOut: &session - ) - - guard error == errSecSuccess, - let session = session else { - throw ConfigurationError.cannotCreateSession - } - - let propertyDictionary = [ - kVTCompressionPropertyKey_PixelTransferProperties: [ - kVTPixelTransferPropertyKey_ScalingMode: kVTScalingMode_Normal - ], - kVTCompressionPropertyKey_ProfileLevel: kVTProfileLevel_H264_Baseline_AutoLevel, - kVTCompressionPropertyKey_MaxKeyFrameInterval: config.expectedFrameRate, - kVTCompressionPropertyKey_ExpectedFrameRate: config.expectedFrameRate, - kVTCompressionPropertyKey_AverageBitRate: config.averageBitRate, - kVTCompressionPropertyKey_RealTime: config.isRealTime, - kVTCompressionPropertyKey_MaximizePowerEfficiency: true, - kVTCompressionPropertyKey_Quality: config.quality - ] as CFDictionary - - guard VTSessionSetProperties(session, propertyDictionary: propertyDictionary) == noErr else { - throw ConfigurationError.cannotSetProperties - } - - guard VTCompressionSessionPrepareToEncodeFrames(session) == noErr else { - throw ConfigurationError.cannotPrepareToEncode - } - - print("VTCompressSession is ready to use") - } - - public func updateEncoderSettings(newBitrate: Int, newFrameRate: Int? = nil) throws { - guard let session = session else { - throw ConfigurationError.cannotSetProperties - } - - var propertyDict: [CFString: Any] = [ - kVTCompressionPropertyKey_AverageBitRate: newBitrate - ] - - if let frameRate = newFrameRate { - propertyDict[kVTCompressionPropertyKey_ExpectedFrameRate] = frameRate - propertyDict[kVTCompressionPropertyKey_MaxKeyFrameInterval] = frameRate - } - - let cfDict = propertyDict as CFDictionary - guard VTSessionSetProperties(session, propertyDictionary: cfDict) == noErr else { - throw ConfigurationError.cannotSetProperties - } - - print("[H264Encoder] Updated settings: bitrate=\(newBitrate) bps" + - (newFrameRate.map { ", frameRate=\($0)" } ?? "")) - } - - public func invalidateCompressionSession() { - guard let session = session else { - return - } - - VTCompressionSessionCompleteFrames(session, untilPresentationTimeStamp: .invalid) - VTCompressionSessionInvalidate(session) - } - - // swiftlint:disable closure_parameter_position - private var encodingOutputCallback: VTCompressionOutputCallback = { ( - outputCallbackRefCon: UnsafeMutableRawPointer?, - _: UnsafeMutableRawPointer?, - status: OSStatus, - flags: VTEncodeInfoFlags, - sampleBuffer: CMSampleBuffer? - ) in - // swiftlint:enable closure_parameter_position - guard let sampleBuffer = sampleBuffer else { - print("nil buffer") - return - } - guard let refcon: UnsafeMutableRawPointer = outputCallbackRefCon else { - print("nil pointer") - return - } - guard status == noErr else { - print("encoding failed") - return - } - guard CMSampleBufferDataIsReady(sampleBuffer) else { - print("CMSampleBuffer is not ready to use") - return - } - guard flags != VTEncodeInfoFlags.frameDropped else { - print("frame dropped") - return - } - - let encoder: H264Encoder = Unmanaged.fromOpaque(refcon).takeUnretainedValue() - - if sampleBuffer.isKeyFrame { - encoder.extractSPSAndPPS(from: sampleBuffer) - } - - var dataBuffer: CMBlockBuffer? - if #available(iOS 13.0, *) { - dataBuffer = sampleBuffer.dataBuffer - } else { - dataBuffer = CMSampleBufferGetDataBuffer(sampleBuffer) - } - guard let dataBuffer = dataBuffer else { return } - - var totalLength: Int = 0 - var dataPointer: UnsafeMutablePointer? - let error = CMBlockBufferGetDataPointer( - dataBuffer, - atOffset: 0, - lengthAtOffsetOut: nil, - totalLengthOut: &totalLength, - dataPointerOut: &dataPointer - ) - - guard error == kCMBlockBufferNoErr, - let dataPointer = dataPointer else { return } - - // emit a timing SEI before the frame's VCL NAL units - let pts = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - if pts.isValid { - encoder.naluHandling?(H264Encoder.createTimingSEI(pts: pts)) - } - - var packageStartIndex = 0 - - while packageStartIndex < totalLength { - var nextNALULength: UInt32 = 0 - memcpy(&nextNALULength, dataPointer.advanced(by: packageStartIndex), 4) - nextNALULength = CFSwapInt32BigToHost(nextNALULength) - guard packageStartIndex + 4 + Int(nextNALULength) <= totalLength else { break } - - let naluLength = Int(nextNALULength) - var naluData = Data(capacity: 4 + naluLength) - naluData.append(naluStartCode) - dataPointer.advanced(by: packageStartIndex + 4) - .withMemoryRebound(to: UInt8.self, capacity: naluLength) { ptr in - naluData.append(ptr, count: naluLength) - } - - packageStartIndex += (4 + naluLength) - - encoder.naluHandling?(naluData) - } - } - - private func extractSPSAndPPS(from sampleBuffer: CMSampleBuffer) { - guard let description = CMSampleBufferGetFormatDescription(sampleBuffer) else { return } - - var parameterSetCount = 0 - CMVideoFormatDescriptionGetH264ParameterSetAtIndex( - description, - parameterSetIndex: 0, - parameterSetPointerOut: nil, - parameterSetSizeOut: nil, - parameterSetCountOut: ¶meterSetCount, - nalUnitHeaderLengthOut: nil - ) - guard parameterSetCount == 2 else { return } - - var spsSize: Int = 0 - var sps: UnsafePointer? - - CMVideoFormatDescriptionGetH264ParameterSetAtIndex( - description, - parameterSetIndex: 0, - parameterSetPointerOut: &sps, - parameterSetSizeOut: &spsSize, - parameterSetCountOut: nil, - nalUnitHeaderLengthOut: nil - ) - - var ppsSize: Int = 0 - var pps: UnsafePointer? - - CMVideoFormatDescriptionGetH264ParameterSetAtIndex( - description, - parameterSetIndex: 1, - parameterSetPointerOut: &pps, - parameterSetSizeOut: &ppsSize, - parameterSetCountOut: nil, - nalUnitHeaderLengthOut: nil - ) - - guard let sps = sps, - let pps = pps else { return } - - for (ptr, size) in [(sps, spsSize), (pps, ppsSize)] { - var data = Data(capacity: 4 + size) - data.append(H264Encoder.naluStartCode) - data.append(ptr, count: size) - naluHandling?(data) - } - } - - // build a SEI NAL unit (user_data_unregistered) carrying the presentation timestamp - // in microseconds. the UUID prefix lets decoders identify and extract the timecode. - static func createTimingSEI(pts: CMTime) -> Data { - let microseconds = UInt64(CMTimeGetSeconds(pts) * 1_000_000) - - // sei_payload: 16-byte UUID + 8-byte big-endian timestamp - var payload = Data(timingUUID) - var ts = microseconds.bigEndian - payload.append(Data(bytes: &ts, count: 8)) - - // sei_message: payloadType(5) + payloadSize(24) + payload - // then rbsp_trailing_bits - var rbsp = Data() - rbsp.append(5) // payloadType = user_data_unregistered - rbsp.append(UInt8(payload.count)) // payloadSize = 24 - rbsp.append(payload) - rbsp.append(0x80) // rbsp_trailing_bits - - let ebsp = addEmulationPrevention(rbsp) - - var nalu = Data([0x06]) // nal_unit_type = 6 (SEI), nal_ref_idc = 0 - nalu.append(ebsp) - - return naluStartCode + nalu - } - - // insert 0x03 emulation prevention bytes where the RBSP contains - // sequences that could be mistaken for start codes (00 00 00, 00 00 01, 00 00 02, 00 00 03) - private static func addEmulationPrevention(_ data: Data) -> Data { - var result = Data() - var zeroCount = 0 - for byte in data { - if zeroCount >= 2 && byte <= 0x03 { - result.append(0x03) - zeroCount = 0 - } - result.append(byte) - zeroCount = (byte == 0x00) ? zeroCount + 1 : 0 - } - return result - } - - public func encode( - sampleBuffer: CMSampleBuffer, - context: CIContext, - orientation: CGImagePropertyOrientation - ) { - guard let session = session, - let imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer), - let rotatedPixelBuffer = imageBuffer.rotate(context: context, orientation: orientation) - else { - return - } - - let timeStamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer) - let duration = CMSampleBufferGetDuration(sampleBuffer) - - VTCompressionSessionEncodeFrame( - session, - imageBuffer: rotatedPixelBuffer, - presentationTimeStamp: timeStamp, - duration: duration, - frameProperties: nil, - sourceFrameRefcon: nil, - infoFlagsOut: nil - ) - } - - public func encode( - imageBuffer: CVImageBuffer, - timestamp: CMTime, - context: CIContext, - orientation: CGImagePropertyOrientation - ) { - guard let session = session, - let rotatedPixelBuffer = imageBuffer.rotate(context: context, orientation: orientation) - else { - return - } - - VTCompressionSessionEncodeFrame( - session, - imageBuffer: rotatedPixelBuffer, - presentationTimeStamp: timestamp, - duration: CMTime.invalid, - frameProperties: nil, - sourceFrameRefcon: nil, - infoFlagsOut: nil - ) - } - - public func encode( - pixelBuffer: CVPixelBuffer, - timestamp: CMTime - ) { - guard let session = session else { return } - - VTCompressionSessionEncodeFrame( - session, - imageBuffer: pixelBuffer, - presentationTimeStamp: timestamp, - duration: CMTime.invalid, - frameProperties: nil, - sourceFrameRefcon: nil, - infoFlagsOut: nil - ) - } - - public func encode( - cgImage: CGImage, - timestamp: CMTime, - context: CIContext, - targetSize: CGSize? = nil, - pool: CVPixelBufferPool? = nil - ) { - guard let pixelBuffer = cgImage.toPixelBuffer( - context: context, - targetSize: targetSize, - pool: pool - ) else { - return - } - - encode(pixelBuffer: pixelBuffer, timestamp: timestamp) - } -} diff --git a/h264-codec/Tests/h264-codecTests/h264_codecTests.swift b/h264-codec/Tests/h264-codecTests/h264_codecTests.swift deleted file mode 100644 index 3ed9902..0000000 --- a/h264-codec/Tests/h264-codecTests/h264_codecTests.swift +++ /dev/null @@ -1,12 +0,0 @@ -import XCTest -@testable import H264Codec - -final class H264CodecTests: XCTestCase { - func testExample() throws { - // XCTest Documentation - // https://developer.apple.com/documentation/xctest - - // Defining Test Cases and Test Methods - // https://developer.apple.com/documentation/xctest/defining_test_cases_and_test_methods - } -} diff --git a/opus-codec/.gitignore b/opus-codec/.gitignore deleted file mode 100644 index 0023a53..0000000 --- a/opus-codec/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -.DS_Store -/.build -/Packages -xcuserdata/ -DerivedData/ -.swiftpm/configuration/registries.json -.swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata -.netrc diff --git a/opus-codec/Package.swift b/opus-codec/Package.swift deleted file mode 100644 index edff3a3..0000000 --- a/opus-codec/Package.swift +++ /dev/null @@ -1,110 +0,0 @@ -// swift-tools-version: 5.9 - -import PackageDescription - -let package = Package( - name: "OpusCodec", - products: [ - .library( - name: "OpusCodec", - targets: ["OpusCodec"] - ), - ], - targets: [ - .target( - name: "Copus", - dependencies: [], - exclude: [ - "AUTHORS", - "autogen.sh", - "celt_headers.mk", - "celt_sources.mk", - "celt/arm", - "celt/dump_modes", - "celt/meson.build", - "celt/opus_custom_demo.c", - "celt/tests", - "celt/x86", - "ChangeLog", - "cmake", - "CMakeLists.txt", - "configure.ac", - "COPYING", - "doc", - "include/meson.build", - "LICENSE_PLEASE_READ.txt", - "m4", - "m4/opus-intrinsics.m4", - "Makefile.am", - "Makefile.mips", - "Makefile.unix", - "meson_options.txt", - "meson.build", - "meson", - "NEWS", - "opus_headers.mk", - "opus_sources.mk", - "opus-uninstalled.pc.in", - "opus.m4", - "opus.pc.in", - "README.draft", - "README", - "releases.sha2", - "scripts/dump_rnn.py", - "scripts/rnn_train.py", - "silk_headers.mk", - "silk_sources.mk", - "silk/arm", - "silk/fixed", - "silk/meson.build", - "silk/mips", - "silk/tests", - "silk/x86", - "src/meson.build", - "src/opus_compare.c", - "src/opus_demo.c", - "src/repacketizer_demo.c", - "tests", - "training", - "update_version", - "win32", - ], - publicHeadersPath: "include", - cSettings: [ - .headerSearchPath("."), - .headerSearchPath("celt"), - .headerSearchPath("celt/x86"), - .headerSearchPath("silk"), - .headerSearchPath("silk/float"), - - .define("OPUS_BUILD"), - .define("VAR_ARRAYS", to: "1"), - .define("FLOATING_POINT"), // Enable Opus floating-point mode - - .define("HAVE_DLFCN_H", to: "1"), - .define("HAVE_INTTYPES_H", to: "1"), - .define("HAVE_LRINT", to: "1"), - .define("HAVE_LRINTF", to: "1"), - .define("HAVE_MEMORY_H", to: "1"), - .define("HAVE_STDINT_H", to: "1"), - .define("HAVE_STDLIB_H", to: "1"), - .define("HAVE_STRING_H", to: "1"), - .define("HAVE_STRINGS_H", to: "1"), - .define("HAVE_SYS_STAT_H", to: "1"), - .define("HAVE_SYS_TYPES_H", to: "1"), - .define("HAVE_UNISTD_H", to: "1"), - ] - ), - .target( - name: "OpusEncoder", - dependencies: ["Copus"], - path: "Sources/OpusEncoder", - publicHeadersPath: "include", - ), - .target( - name: "OpusCodec", - dependencies: ["OpusEncoder"], - path: "Sources/OpusCodec", - ), - ] -) diff --git a/opus-codec/Sources/Copus b/opus-codec/Sources/Copus deleted file mode 160000 index 101a71e..0000000 --- a/opus-codec/Sources/Copus +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 101a71e03bbf860aaafb7090a0e440675cb27660 diff --git a/opus-codec/Sources/OpusCodec/AudioFormatConverter.swift b/opus-codec/Sources/OpusCodec/AudioFormatConverter.swift deleted file mode 100644 index 29e3f9e..0000000 --- a/opus-codec/Sources/OpusCodec/AudioFormatConverter.swift +++ /dev/null @@ -1,253 +0,0 @@ -import AudioToolbox -import CoreMedia -import Foundation - -final class AudioFormatConverter { - - private var converter: AudioConverterRef? - private var inputFormat: AudioStreamBasicDescription? - - private let outputSampleRate: Double - private let outputChannels: UInt32 - - init(sampleRate: Double, channels: UInt32) { - self.outputSampleRate = sampleRate - self.outputChannels = channels - } - - deinit { - invalidate() - } - - func invalidate() { - if let converter { - AudioConverterDispose(converter) - } - - converter = nil - inputFormat = nil - } - - func convert(sampleBuffer: CMSampleBuffer) -> [Int16]? { - guard let formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer), - let asbdPointer = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription) else { - return nil - } - - let inputASBD = asbdPointer.pointee - // NSLog("[AudioFormatConverter] Input ASBD: rate=\(inputASBD.mSampleRate) channels=\(inputASBD.mChannelsPerFrame) bytesPerFrame=\(inputASBD.mBytesPerFrame) formatFlags=\(inputASBD.mFormatFlags) formatID=\(inputASBD.mFormatID)") - guard inputASBD.mFormatID == kAudioFormatLinearPCM else { - NSLog("[AudioFormatConverter] Input format is not linear PCM") - return nil - } - - if !configureConverterIfNeeded(input: inputASBD) { - return nil - } - - var blockBuffer: CMBlockBuffer? - var requiredSize: Int = 0 - let sizeStatus = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( - sampleBuffer, - bufferListSizeNeededOut: &requiredSize, - bufferListOut: nil, - bufferListSize: 0, - blockBufferAllocator: kCFAllocatorDefault, - blockBufferMemoryAllocator: kCFAllocatorDefault, - flags: 0, - blockBufferOut: &blockBuffer - ) - - if sizeStatus != noErr && sizeStatus != kCMSampleBufferError_ArrayTooSmall { - NSLog("[AudioFormatConverter] Failed to query AudioBufferList size: \(sizeStatus)") - return nil - } - - let rawPointer = UnsafeMutableRawPointer.allocate( - byteCount: requiredSize, - alignment: MemoryLayout.alignment - ) - defer { rawPointer.deallocate() } - - let bufferListPointer = rawPointer.bindMemory( - to: AudioBufferList.self, - capacity: 1 - ) - - let status = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer( - sampleBuffer, - bufferListSizeNeededOut: nil, - bufferListOut: bufferListPointer, - bufferListSize: requiredSize, - blockBufferAllocator: kCFAllocatorDefault, - blockBufferMemoryAllocator: kCFAllocatorDefault, - flags: 0, - blockBufferOut: &blockBuffer - ) - - guard status == noErr else { - NSLog("[AudioFormatConverter] CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer failed: \(status)") - return nil - } - - let bufferList = UnsafeMutableAudioBufferListPointer(bufferListPointer) - - let inputFrames = CMSampleBufferGetNumSamples(sampleBuffer) - if inputFrames <= 0 { - NSLog("[AudioFormatConverter] Input sample buffer has no frames") - return nil - } - - let ratio = outputSampleRate / Double(inputASBD.mSampleRate) - let outputFrames = max(1, Int(Double(inputFrames) * ratio)) - - var outputPCM = [Int16](repeating: 0, count: outputFrames * Int(outputChannels)) - let outputByteCount = outputPCM.count * MemoryLayout.size - var ioOutputDataPacketSize = UInt32(outputFrames) - - var inputContext = InputContext( - sourceBufferList: bufferList, - bytesPerFrame: inputASBD.mBytesPerFrame, - frameCount: UInt32(inputFrames), - frameOffset: 0 - ) - - let convertStatus = withUnsafeMutablePointer(to: &inputContext) { contextPtr in - outputPCM.withUnsafeMutableBytes { rawBuffer -> OSStatus in - guard let baseAddress = rawBuffer.baseAddress else { - return kAudio_ParamError - } - - var outBufferList = AudioBufferList( - mNumberBuffers: 1, - mBuffers: AudioBuffer( - mNumberChannels: outputChannels, - mDataByteSize: UInt32(outputByteCount), - mData: baseAddress - ) - ) - - return AudioConverterFillComplexBuffer( - converter!, - audioConverterInputDataProc, - contextPtr, - &ioOutputDataPacketSize, - &outBufferList, - nil - ) - } - } - - if convertStatus != noErr { - NSLog("[AudioFormatConverter] PCM conversion failed: \(convertStatus)") - return nil - } - - let producedSamples = Int(ioOutputDataPacketSize) * Int(outputChannels) - if producedSamples <= 0 { - NSLog("[AudioFormatConverter] PCM conversion produced no samples") - return nil - } - - outputPCM.removeLast(outputPCM.count - producedSamples) - return outputPCM - } - - private func configureConverterIfNeeded(input: AudioStreamBasicDescription) -> Bool { - if let existing = inputFormat, isSameFormat(existing, input) { - return true - } - - if let converter { - AudioConverterDispose(converter) - } - converter = nil - inputFormat = input - - var output = AudioStreamBasicDescription( - mSampleRate: outputSampleRate, - mFormatID: kAudioFormatLinearPCM, - mFormatFlags: kLinearPCMFormatFlagIsSignedInteger | kLinearPCMFormatFlagIsPacked, - mBytesPerPacket: 2, - mFramesPerPacket: 1, - mBytesPerFrame: 2, - mChannelsPerFrame: outputChannels, - mBitsPerChannel: 16, - mReserved: 0 - ) - - var converterRef: AudioConverterRef? - var inputASBD = input - let status = AudioConverterNew(&inputASBD, &output, &converterRef) - guard status == noErr, let converterRef else { - NSLog("[AudioFormatConverter] Failed to create PCM converter: \(status)") - return false - } - - var quality = UInt32(kAudioConverterQuality_Medium) - let qualityStatus = AudioConverterSetProperty( - converterRef, - kAudioConverterSampleRateConverterQuality, - UInt32(MemoryLayout.size(ofValue: quality)), - &quality - ) - if qualityStatus != noErr { - NSLog("[AudioFormatConverter] Failed to set converter quality: \(qualityStatus)") - } - - converter = converterRef - return true - } - - private func isSameFormat(_ lhs: AudioStreamBasicDescription, _ rhs: AudioStreamBasicDescription) -> Bool { - lhs.mSampleRate == rhs.mSampleRate && - lhs.mFormatID == rhs.mFormatID && - lhs.mFormatFlags == rhs.mFormatFlags && - lhs.mChannelsPerFrame == rhs.mChannelsPerFrame && - lhs.mBitsPerChannel == rhs.mBitsPerChannel && - lhs.mBytesPerFrame == rhs.mBytesPerFrame && - lhs.mBytesPerPacket == rhs.mBytesPerPacket && - lhs.mFramesPerPacket == rhs.mFramesPerPacket - } - - private struct InputContext { - var sourceBufferList: UnsafeMutableAudioBufferListPointer - var bytesPerFrame: UInt32 - var frameCount: UInt32 - var frameOffset: UInt32 - } - - private let audioConverterInputDataProc: AudioConverterComplexInputDataProc = { - _, ioNumberDataPackets, ioData, _, userData in - guard let userData else { - return kAudio_ParamError - } - - let context = userData.assumingMemoryBound(to: InputContext.self) - if context.pointee.frameOffset >= context.pointee.frameCount { - ioNumberDataPackets.pointee = 0 - return noErr - } - - let framesRemaining = context.pointee.frameCount - context.pointee.frameOffset - let framesToCopy = min(framesRemaining, ioNumberDataPackets.pointee) - ioNumberDataPackets.pointee = framesToCopy - - let sourceList = context.pointee.sourceBufferList - let destList = UnsafeMutableAudioBufferListPointer(ioData) - destList.count = sourceList.count - - let byteOffset = Int(context.pointee.frameOffset * context.pointee.bytesPerFrame) - let byteCount = Int(framesToCopy * context.pointee.bytesPerFrame) - - for index in 0.. Void)? - - private var encoder: OpusEncoderRef? - private var audioConverter: AudioFormatConverter - private var pendingPCM: [Int16] = [] - - private let outputSampleRate: Double = 48_000 - private let outputChannels: UInt32 = 1 - private let frameSize: Int = 960 // 20 ms @ 48 kHz - private let maxPacketSize: Int = 4000 - private var bitRate: Int - - public init(bitRate: Int = 64_000) { - self.bitRate = bitRate - self.audioConverter = AudioFormatConverter(sampleRate: outputSampleRate, channels: outputChannels) - } - - deinit { - invalidate() - } - - public func invalidate() { - if let encoder { - OpusEncoderDestroy(encoder) - } - - encoder = nil - audioConverter.invalidate() - pendingPCM.removeAll(keepingCapacity: true) - } - - public func updateBitRate(_ newBitRate: Int) { - bitRate = newBitRate - if let encoder { - let status = OpusEncoderSetBitrate(encoder, Int32(bitRate)) - if status != 0 { - print("[OpusAudioEncoder] Failed to update bitrate: \(status)") - } - } - } - - public func encode(sampleBuffer: CMSampleBuffer) { - guard ensureEncoder() else { - return - } - - guard let pcm = audioConverter.convert(sampleBuffer: sampleBuffer) else { - NSLog("[OpusAudioEncoder] PCM conversion returned nil") - return - } - - pendingPCM.append(contentsOf: pcm) - encodePendingFrames() - } - - private func encodePendingFrames() { - guard let encoder = encoder else { - return - } - - while pendingPCM.count >= frameSize { - let frame = pendingPCM.prefix(frameSize) - let data = frame.withUnsafeBufferPointer { buffer -> Data? in - guard let baseAddress = buffer.baseAddress else { - return nil - } - - var output = [UInt8](repeating: 0, count: maxPacketSize) - let encoded = OpusEncoderEncode( - encoder, - baseAddress, - Int32(frameSize), - &output, - Int32(maxPacketSize) - ) - - guard encoded > 0 else { - NSLog("[OpusAudioEncoder] Opus encode failed: \(encoded)") - return nil - } - - return Data(output.prefix(Int(encoded))) - } - - if let data { - NSLog("[OpusAudioEncoder] Opus frame bytes: \(data.count)") - opusHandling?(data) - } - - pendingPCM.removeFirst(frameSize) - } - } - - private func ensureEncoder() -> Bool { - if encoder != nil { - return true - } - - var error: Int32 = 0 - encoder = OpusEncoderCreate( - Int32(outputSampleRate), - Int32(outputChannels), - Int32(bitRate), - &error - ) - - if error != 0 { - NSLog("[OpusAudioEncoder] Failed to create Opus encoder: \(error)") - encoder = nil - return false - } - return true - } -} diff --git a/opus-codec/Sources/OpusEncoder/OpusEncoder.c b/opus-codec/Sources/OpusEncoder/OpusEncoder.c deleted file mode 100644 index 31103bd..0000000 --- a/opus-codec/Sources/OpusEncoder/OpusEncoder.c +++ /dev/null @@ -1,44 +0,0 @@ -#include "OpusEncoder.h" - -OpusEncoderRef OpusEncoderCreate(int sampleRate, int channels, int bitrate, int *error) { - int err = OPUS_OK; - OpusEncoder *encoder = opus_encoder_create(sampleRate, channels, OPUS_APPLICATION_AUDIO, &err); - if (err != OPUS_OK) { - if (error != NULL) { - *error = err; - } - - return NULL; - } - - opus_encoder_ctl(encoder, OPUS_SET_BITRATE(bitrate)); - if (error != NULL) { - *error = OPUS_OK; - } - - return encoder; -} - -int OpusEncoderEncode(OpusEncoderRef encoder, const int16_t *pcm, int frameSize, uint8_t *data, int maxDataBytes) { - if (encoder == NULL) { - return OPUS_BAD_ARG; - } - - return opus_encode(encoder, (const opus_int16 *)pcm, frameSize, data, maxDataBytes); -} - -int OpusEncoderSetBitrate(OpusEncoderRef encoder, int bitrate) { - if (encoder == NULL) { - return OPUS_BAD_ARG; - } - - return opus_encoder_ctl(encoder, OPUS_SET_BITRATE(bitrate)); -} - -void OpusEncoderDestroy(OpusEncoderRef encoder) { - if (encoder == NULL) { - return; - } - - opus_encoder_destroy(encoder); -} diff --git a/opus-codec/Sources/OpusEncoder/include/OpusEncoder.h b/opus-codec/Sources/OpusEncoder/include/OpusEncoder.h deleted file mode 100644 index d60e46f..0000000 --- a/opus-codec/Sources/OpusEncoder/include/OpusEncoder.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef OPUS_ENCODER_WRAPPER_H -#define OPUS_ENCODER_WRAPPER_H - -#include -#include - -typedef OpusEncoder *OpusEncoderRef; - -OpusEncoderRef OpusEncoderCreate(int sampleRate, int channels, int bitrate, int *error); -void OpusEncoderDestroy(OpusEncoderRef encoder); - -int OpusEncoderEncode( - OpusEncoderRef encoder, - const int16_t *pcm, - int frameSize, - uint8_t *data, - int maxDataBytes -); - -int OpusEncoderSetBitrate(OpusEncoderRef encoder, int bitrate); - -#endif diff --git a/opus-codec/Tests/opus-codecTests/opus_codecTests.swift b/opus-codec/Tests/opus-codecTests/opus_codecTests.swift deleted file mode 100644 index 0508742..0000000 --- a/opus-codec/Tests/opus-codecTests/opus_codecTests.swift +++ /dev/null @@ -1,12 +0,0 @@ -import XCTest -@testable import H264Codec - -final class OpusCodecTests: XCTestCase { - func testExample() throws { - // XCTest Documentation - // https://developer.apple.com/documentation/xctest - - // Defining Test Cases and Test Methods - // https://developer.apple.com/documentation/xctest/defining_test_cases_and_test_methods - } -}