diff --git a/Package.swift b/Package.swift index 9ed4027..f6da5c0 100644 --- a/Package.swift +++ b/Package.swift @@ -21,6 +21,9 @@ let package = Package( name: "Lumen", dependencies: [ .product(name: "FFmpegKit", package: "FFmpegKit"), + // The core has no Mac Catalyst slices; the Atmos activator is for Apple TV's HDMI output. + .product(name: "SpatialAudioBridge", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), + .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS])), // .product(name: "Libass", package: "FFmpegKit"), // .product(name: "Libmpv", package: "FFmpegKit"), "DisplayCriteria", @@ -43,4 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "2.0.0"), ] diff --git a/README.md b/README.md index 51ab7f2..e077b5c 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- TrueHD and DTS are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone +- **TrueHD Atmos and DTS:X play as APAC** on tvOS, iOS and macOS 26 and later: [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and on tvOS and iOS a silent Dolby Digital Plus Atmos stream plays alongside, which switches a receiver to Atmos. A receiver showed Atmos through Lumen on Apple TV, and AirPods Pro on an iPhone played it as spatial audio with head tracking +- Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode - On tvOS, frame rate and dynamic range are handed to the display through `AVDisplayCriteria` — so the TV can switch mode to match the content, when the viewer has Match Content enabled @@ -80,7 +81,8 @@ There are two floors, and they are not the same one: Scrub thumbnails are tvOS-only, at any version. -- Swift 5.9+ / Xcode 15+ +- Xcode 26 or later: SpatialAudioBridge, which converts TrueHD Atmos and DTS:X, needs Swift 6 tools and the tvOS, iOS and macOS 26 SDKs to build. Lumen itself still compiles in the Swift 5 language mode +- SpatialAudioBridge's core is universal for macOS and the iOS simulator; generic Apple TV simulator builds need `ARCHS=arm64`. Mac Catalyst builds leave it out and keep the FLAC path ## Installation @@ -190,3 +192,5 @@ Lumen grew out of [**KSPlayer**](https://github.com/kingslay/KSPlayer) by [kings ## License GPL-3.0 — see [`LICENSE`](./LICENSE). + +Apps that ship Lumen show `SpatialAudioBridgeNotices.text`, the license notices of SpatialAudioBridge's core, in their acknowledgements. diff --git a/Sources/Lumen/AVPlayer/KSPlayerLayer.swift b/Sources/Lumen/AVPlayer/KSPlayerLayer.swift index 359d8b5..d6411a8 100644 --- a/Sources/Lumen/AVPlayer/KSPlayerLayer.swift +++ b/Sources/Lumen/AVPlayer/KSPlayerLayer.swift @@ -171,18 +171,24 @@ open class KSPlayerLayer: NSObject { } } - private lazy var timer: Timer = .scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in - guard let self, self.player.isReadyToPlay else { - return - } - self.delegate?.player(layer: self, currentTime: self.player.currentPlaybackTime, totalTime: self.player.duration) - if self.player.playbackState == .playing, self.player.loadState == .playable, self.state == .buffering { - // 一个兜底保护，正常不能走到这里 - self.state = .bufferFinished - } - if self.player.isPlaying { - MPNowPlayingInfoCenter.default().nowPlayingInfo?[MPNowPlayingInfoPropertyElapsedPlaybackTime] = self.player.currentPlaybackTime + private var playbackTimer: Timer? + private var timer: Timer { + if let playbackTimer { return playbackTimer } + let timer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in + guard let self, self.player.isReadyToPlay else { + return + } + self.delegate?.player(layer: self, currentTime: self.player.currentPlaybackTime, totalTime: self.player.duration) + if self.player.playbackState == .playing, self.player.loadState == .playable, self.state == .buffering { + // 一个兜底保护，正常不能走到这里 + self.state = .bufferFinished + } + if self.player.isPlaying { + MPNowPlayingInfoCenter.default().nowPlayingInfo?[MPNowPlayingInfoPropertyElapsedPlaybackTime] = self.player.currentPlaybackTime + } } + playbackTimer = timer + return timer } private var urls = [URL]() @@ -239,7 +245,8 @@ open class KSPlayerLayer: NSObject { } deinit { - timer.invalidate() + // Cleanup must not initialize a timer whose closure captures a deallocating self. + playbackTimer?.invalidate() if #available(iOS 15.0, tvOS 15.0, macOS 12.0, *) { player.pipController?.contentSource = nil } diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 75cb2d1..63aa6c3 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -25,6 +25,8 @@ public final class MEPlayerItem: Sendable { private let remuxSession: ProAVRemuxSession? private var remuxTranscoder: ProAVAudioTranscoder? private var remuxTranscodeStreamIndex: Int? + private var remuxSpatialAudio: ProAVSpatialAudioTranscoder? + private var remuxAudioEndTime: TimeInterval? private var remuxCompleted = false private var remuxHeaderWritten = false private var remuxMoovWritten = false @@ -357,7 +359,19 @@ extension MEPlayerItem { let audioAssetTrack = audios.first { preferredAudioTrackID == nil ? $0.isEnabled : $0.trackID == preferredAudioTrackID } ?? audios.first { $0.isEnabled } ?? audios.first if let audioAssetTrack, let inputAudioStream = formatCtx.pointee.streams[Int(audioAssetTrack.trackID)] { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) - if strategy.copiesBitstream { + let spatialAudioPolicy = session.spatialAudioPolicy + session.spatialAudioWithheld = spatialAudioPolicy.isEnabled && spatialAudioPolicy.isExternalPlaybackActive && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) + var triesSpatialAudio = spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID) + if triesSpatialAudio, probeRefusesSpatialAudio(stream: inputAudioStream, formatCtx: formatCtx) { + triesSpatialAudio = false + // Later launches of this source skip the probe. + session.spatialAudioPolicy.refusedTrackIDs.insert(audioAssetTrack.trackID) + } + guard error == nil else { return } + if triesSpatialAudio, let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { + remuxSpatialAudio = spatialAudio + audioSignaling = spatialAudio.signaling + } else if strategy.copiesBitstream { if let outputAudioStream = avformat_new_stream(outputFormatCtx, nil) { avcodec_parameters_copy(outputAudioStream.pointee.codecpar, inputAudioStream.pointee.codecpar) outputAudioStream.pointee.codecpar.pointee.codec_tag = 0 @@ -381,7 +395,8 @@ extension MEPlayerItem { } else { KSLog("ProAV audio track skipped: flac transcode unavailable") } - if streamMapping[Int(audioAssetTrack.trackID)] != nil { + if streamMapping[Int(audioAssetTrack.trackID)] != nil || remuxSpatialAudio != nil { + remuxAudioEndTime = Self.audioEndTime(stream: inputAudioStream.pointee, sourceStart: startTime.seconds, formatName: options.formatName) audios.forEach { $0.isEnabled = $0.trackID == audioAssetTrack.trackID } } } @@ -412,6 +427,30 @@ extension MEPlayerItem { outputPacket = av_packet_alloc() } + /// Whether the library will refuse a track the conversion would take, such as DTS-HD MA without + /// DTS:X, so it plays through FLAC from the start instead of failing the launch. The probe reads + /// ahead, so it needs a seekable source, and a launch from the start rewinds afterwards; the read + /// thread seeks a later start itself. A launch that cannot rewind fails rather than skip the start. + private func probeRefusesSpatialAudio(stream: UnsafeMutablePointer, formatCtx: UnsafeMutablePointer) -> Bool { + guard seekable, let codecpar = stream.pointee.codecpar, ProAVSpatialAudioTranscoder.converts(codecpar: codecpar.pointee) else { + return false + } + let refusal = ProAVSpatialAudioTranscoder.probeRefusal(stream: stream, formatContext: formatCtx) + if options.startPlayTime <= 0 { + let start = seekByBytes || formatCtx.pointee.start_time == Int64.min ? 0 : formatCtx.pointee.start_time + let result = avformat_seek_file(formatCtx, -1, Int64.min, start, Int64.max, seekByBytes ? AVSEEK_FLAG_BYTE : 0) + if result < 0 { + error = NSError(description: "ProAV spatial audio probe could not rewind: \(result)") + return false + } + } + guard let refusal else { + return false + } + KSLog(level: .info, "ProAV spatial audio unsupported: \(refusal); playing FLAC") + return true + } + private func overrideDolbyVisionConfigurationRecord(codecpar: UnsafeMutablePointer) -> Bool { let recordSize = MemoryLayout.size var source = [UInt8]() @@ -437,10 +476,20 @@ extension MEPlayerItem { private func writeProAVPacket(corePacket: UnsafeMutablePointer, outputFormatCtx: UnsafeMutablePointer, formatCtx: UnsafeMutablePointer, session: ProAVRemuxSession) { guard !session.isFailed else { return } let index = Int(corePacket.pointee.stream_index) + if let remuxSpatialAudio, remuxSpatialAudio.streamIndex == index { + remuxSpatialAudio.append(packet: corePacket) + if let inputStream = formatCtx.pointee.streams[index] { + trackRemuxAudioTime(packet: corePacket, stream: inputStream, session: session) + } + return + } guard let outputIndex = streamMapping[index], let inputStream = formatCtx.pointee.streams[index], let outputStream = outputFormatCtx.pointee.streams[outputIndex] else { return } + if outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_AUDIO { + trackRemuxAudioTime(packet: corePacket, stream: inputStream, session: session) + } if index == remuxTranscodeStreamIndex { guard let remuxTranscoder else { return } let succeeded = remuxTranscoder.transcode(packet: corePacket) { encodedPacket in @@ -496,6 +545,11 @@ extension MEPlayerItem { outputPacket.pointee.stream_index = Int32(outputIndex) av_packet_rescale_ts(outputPacket, inputStream.pointee.time_base, outputStream.pointee.time_base) outputPacket.pointee.pos = -1 + if let remuxSpatialAudio, !remuxSpatialAudio.hasTimeline, outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_VIDEO, + let timeline = ProAVSpatialAudioTranscoder.timeline(pts: outputPacket.pointee.pts, dts: outputPacket.pointee.dts, duration: outputPacket.pointee.duration, videoDelay: outputStream.pointee.codecpar.pointee.video_delay, timebase: Timebase(outputStream.pointee.time_base)) + { + remuxSpatialAudio.begin(origin: timeline.origin, videoStart: timeline.videoStart) + } let ret = av_write_frame(outputFormatCtx, outputPacket) if ret < 0 { KSLog("ProAV remux can not av_write_frame") @@ -505,6 +559,32 @@ extension MEPlayerItem { av_packet_unref(outputPacket) } + private func trackRemuxAudioTime(packet: UnsafeMutablePointer, stream: UnsafeMutablePointer, session: ProAVRemuxSession) { + let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts + guard timestamp != Int64.min else { return } + let timebase = Timebase(stream.pointee.time_base) + session.trackAudioTime(seconds: timebase.cmtime(for: timestamp).seconds, duration: timebase.cmtime(for: max(packet.pointee.duration, 0)).seconds, streamEnd: remuxAudioEndTime) + } + + /// Matroska's DURATION tag is the track's final timestamp, including the last packet's duration. + static func audioEndTime(stream: AVStream, sourceStart: TimeInterval, formatName: String) -> TimeInterval? { + if formatName.split(separator: ",").contains(where: { $0 == "matroska" || $0 == "webm" }), + let tag = toDictionary(stream.metadata)["DURATION"] + { + let parts = tag.split(separator: ":", omittingEmptySubsequences: false) + if parts.count == 3, let hours = Double(parts[0]), let minutes = Double(parts[1]), let seconds = Double(parts[2]), + hours >= 0, minutes >= 0, minutes < 60, seconds >= 0, seconds < 60 + { + let end = hours * 3600 + minutes * 60 + seconds + if end.isFinite, end > 0 { return end } + } + } + guard stream.duration > 0, stream.time_base.num > 0, stream.time_base.den > 0 else { return nil } + let timebase = Timebase(stream.time_base) + let start = stream.start_time == Int64.min ? sourceStart : timebase.cmtime(for: stream.start_time).seconds + return start + timebase.cmtime(for: stream.duration).seconds + } + private static func requiresParsedHeader(codecID: AVCodecID) -> Bool { codecID == AV_CODEC_ID_AC3 || codecID == AV_CODEC_ID_EAC3 } @@ -540,6 +620,12 @@ extension MEPlayerItem { private func finishRemux(reachedEnd: Bool) { guard let remuxSession, !remuxCompleted else { return } remuxCompleted = true + if reachedEnd, !remuxSession.isFailed { + remuxSpatialAudio?.finish() + } else { + remuxSpatialAudio?.cancel() + } + remuxSpatialAudio = nil if let outputFormatCtx, remuxHeaderWritten { if reachedEnd, let remuxTranscoder, let streamIndex = remuxTranscodeStreamIndex, let outputIndex = streamMapping[streamIndex], @@ -1079,7 +1165,15 @@ extension MEPlayerItem: MediaPlayback { extension MEPlayerItem: CodecCapacityDelegate { func codecDidChangeCapacity() { - guard remuxSession == nil else { + if let remuxSession { + // The remux reads ahead of playback by the buffer duration, as the decoding path buffers. + if let ahead = remuxSession.remuxedAhead(sourceStart: startTime.seconds) { + if ahead > options.maxBufferDuration { + pause() + } else if ahead < options.maxBufferDuration / 2 { + resume() + } + } return } let loadingState = options.playable(capacitys: videoAudioTracks, isFirst: isFirst, isSeek: isSeek) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 9240196..722fc64 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -3,6 +3,9 @@ import Foundation #if os(tvOS) || os(xrOS) import DisplayCriteria #endif +#if (os(tvOS) || os(iOS)) && canImport(SpatialAudioPlayback) +import SpatialAudioPlayback +#endif #if canImport(UIKit) import UIKit #else @@ -30,6 +33,18 @@ public final class ProAVPlayer { private var delegateProxy: ProAVLaunchDelegateProxy? private var pendingSourceSwitch: PendingSourceSwitch? private var preferredAudioTrackID: Int32? + /// Audio tracks of this source that the library or a launch's probe refused; they keep the FLAC path. + private var refusedSpatialAudioTrackIDs = Set() + /// Whether playback resumes after an in-place restart, for the FLAC fallback or AirPlay, of a launch + /// that was already playing. + private var fallbackResumesPlayback: Bool? + private var playbackTimeObserver: Any? + private var externalPlaybackObservation: NSKeyValueObservation? + private var externalPlaybackObservationGeneration = 0 + /// Keep the observed route across item removal, which can clear AVPlayer's active flag. + private var externalPlaybackActive = false + /// The AtmosOutputActivator while a launch plays APAC on tvOS or iOS; releasing it detaches it. + private var atmosOutput: AnyObject? private var reportedReady = false private var didFail = false public weak var delegate: MediaPlayerDelegate? @@ -47,6 +62,7 @@ public final class ProAVPlayer { let url: URL let options: KSOptions let startOffset: TimeInterval + let spatialAudio: ProAVRemuxSession.SpatialAudioPolicy let completion: (Bool) -> Void var innerSwitchStarted = false } @@ -59,6 +75,47 @@ public final class ProAVPlayer { ProAVPlayer.purgeWorkspaces() workspaceURL = ProAVPlayer.workspaceRoot.appendingPathComponent(UUID().uuidString) innerPlayer.delegate = self + // The remux reads only so far ahead of this position. + playbackTimeObserver = innerPlayer.player.addPeriodicTimeObserver(forInterval: CMTime(value: 1, timescale: 1), queue: .main) { [weak self] _ in + MainActor.assumeIsolated { + guard let self else { return } + self.session?.playbackTime = self.currentPlaybackTime + } + } + observeExternalPlayback() + } + + private func observeExternalPlayback() { + #if !os(xrOS) + let generation = externalPlaybackObservationGeneration + externalPlaybackObservation = innerPlayer.player.observe(\.isExternalPlaybackActive) { [weak self] player, _ in + let isActive = player.isExternalPlaybackActive + Task { @MainActor in + guard let self, self.externalPlaybackObservationGeneration == generation else { return } + self.externalPlaybackDidChange(isActive: isActive) + } + } + #endif + } + + /// Replacing an item must not turn a teardown notification into a route change. The generation + /// also drops notifications already queued by the previous observation. + private func updateInnerPlayerItem(_ update: () -> Void) { + // Capture a real disconnect that may still be queued on the main actor. Once the item is + // gone, a false property value can instead be a side effect of our own teardown. + if innerPlayer.player.currentItem != nil { + externalPlaybackActive = innerPlayer.isExternalPlaybackActive + } + externalPlaybackObservation?.invalidate() + externalPlaybackObservation = nil + externalPlaybackObservationGeneration += 1 + update() + // Attachment can activate AirPlay synchronously while observation is suspended. Only an + // observed route change may clear the saved state; item removal's false value cannot. + if innerPlayer.isExternalPlaybackActive { + externalPlaybackActive = true + } + observeExternalPlayback() } private static var workspaceRoot: URL { @@ -73,13 +130,18 @@ public final class ProAVPlayer { } } - private func makeLaunch(url: URL, options: KSOptions, at time: TimeInterval, audioTrackID: Int32?) -> RemuxLaunch { + private func makeLaunch(url: URL, options: KSOptions, at time: TimeInterval, audioTrackID: Int32?, spatialAudio: ProAVRemuxSession.SpatialAudioPolicy) -> RemuxLaunch { launchIndex += 1 let directoryName = "launch\(launchIndex)" let directory = workspaceURL.appendingPathComponent(directoryName) let configuration = ProAVRemuxSession.Configuration(directory: directory, targetSegmentDuration: ProAVPlayer.segmentDuration, minimumSegmentsBeforeReady: ProAVPlayer.minimumSegmentsBeforeReady) let session = ProAVRemuxSession(configuration: configuration) session.preferredAudioTrackID = audioTrackID + var spatialAudio = spatialAudio + // An AirPlay receiver plays the stream itself, so the APAC rendition is left out. + spatialAudio.isExternalPlaybackActive = externalPlaybackActive + session.spatialAudioPolicy = spatialAudio + session.playbackTime = time options.startPlayTime = time let item = MEPlayerItem(url: url, options: options, remuxSession: session) let proxy = ProAVLaunchDelegateProxy(target: self, launch: launchIndex) @@ -87,8 +149,9 @@ public final class ProAVPlayer { return RemuxLaunch(index: launchIndex, directoryName: directoryName, session: session, item: item, proxy: proxy) } - private func startRemux(at time: TimeInterval) { - let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: preferredAudioTrackID) + private func startRemux(at time: TimeInterval, spatialAudioEnabled: Bool = true) { + let spatialAudio = ProAVRemuxSession.SpatialAudioPolicy(refusedTrackIDs: refusedSpatialAudioTrackIDs, isEnabled: spatialAudioEnabled) + let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: preferredAudioTrackID, spatialAudio: spatialAudio) launch.session.onReady = { [weak self, weak session = launch.session, directoryName = launch.directoryName] _ in runOnMainThread { guard let self, let session, self.session === session else { return } @@ -98,7 +161,7 @@ public final class ProAVPlayer { launch.session.onFailure = { [weak self, weak session = launch.session] sessionError in runOnMainThread { guard let self, let session, self.session === session else { return } - self.fail(error: sessionError) + self.sessionDidFail(session, error: sessionError) } } session = launch.session @@ -111,6 +174,7 @@ public final class ProAVPlayer { private func remuxDidBecomeReady(directoryName: String) { guard !didFail else { return } + guard !restartForExternalPlaybackIfNeeded() else { return } applyDisplayCriteria() if let serverBaseURL { attach(baseURL: serverBaseURL, directoryName: directoryName) @@ -131,10 +195,31 @@ public final class ProAVPlayer { } private func attach(baseURL: URL, directoryName: String) { - guard !didFail else { return } + guard !didFail, directoryName == "launch\(activeLaunchIndex)" else { return } + guard !restartForExternalPlaybackIfNeeded() else { return } let masterURL = baseURL.appendingPathComponent(directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) - innerPlayer.replace(url: masterURL, options: options) - innerPlayer.prepareToPlay() + updateSpatialAudioOutput() + updateInnerPlayerItem { + innerPlayer.replace(url: masterURL, options: options) + innerPlayer.prepareToPlay() + } + _ = restartForExternalPlaybackIfNeeded() + } + + /// Plays the activator while the active launch plays APAC on this device: APAC alone doesn't switch + /// a receiver to Dolby Atmos, and AirPods render it flatter without it. + private func updateSpatialAudioOutput() { + #if (os(tvOS) || os(iOS)) && canImport(SpatialAudioPlayback) + guard #available(tvOS 26, iOS 26, *) else { return } + guard session?.playsSpatialAudio == true, !externalPlaybackActive else { + atmosOutput = nil + return + } + guard atmosOutput == nil else { return } + let activator = AtmosOutputActivator() + activator.attach(to: innerPlayer.player) + atmosOutput = activator + #endif } private func applyDisplayCriteria() { @@ -151,6 +236,63 @@ public final class ProAVPlayer { #endif } + private func sessionDidFail(_ session: ProAVRemuxSession, error: NSError) { + guard !didFail else { return } + guard let failure = session.spatialAudioFailure else { + fail(error: error) + return + } + KSLog(level: .info, "\(error.localizedDescription); falling back to FLAC") + if case let .refused(trackID) = failure { + refusedSpatialAudioTrackIDs.insert(trackID) + } + restartInPlace(spatialAudioEnabled: false) + } + + private func externalPlaybackDidChange(isActive: Bool) { + externalPlaybackActive = isActive + guard !didFail else { return } + // The activator stops as AirPlay starts, ahead of the restart without APAC. + updateSpatialAudioOutput() + guard isActive else { + // Keep the requested source switch. Its readiness callback also rechecks the route if + // opening has not selected audio yet. + if let pending = pendingSourceSwitch { + if pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: false) { + restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) + } + return + } + _ = restartForExternalPlaybackIfNeeded() + return + } + if session?.playsSpatialAudio == true { + KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") + // Like a seek, the restart abandons a pending switch, which the layer then makes cold, + // without APAC while AirPlay lasts. + _ = restartForExternalPlaybackIfNeeded() + } else if let pending = pendingSourceSwitch, pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: true) { + restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) + } + } + + @discardableResult + private func restartForExternalPlaybackIfNeeded() -> Bool { + guard let session, session.needsSpatialAudioRestart(isExternalPlaybackActive: externalPlaybackActive) else { return false } + restartInPlace(spatialAudioEnabled: session.spatialAudioPolicy.isEnabled) + return true + } + + /// Restarts the active launch at the current position, keeping the play or pause state. + private func restartInPlace(spatialAudioEnabled: Bool) { + let seekCompletion = pendingSeekCompletion + pendingSeekCompletion = nil + // The remux runs ahead of playback, so this can happen while the launch plays. + let resumesPlayback = isReadyToPlay && seekCompletion == nil ? isPlaying : nil + restart(at: currentPlaybackTime, completion: seekCompletion, spatialAudioEnabled: spatialAudioEnabled) + fallbackResumesPlayback = resumesPlayback + } + private func fail(error: NSError) { abortPendingSourceSwitch() guard !didFail else { return } @@ -159,7 +301,8 @@ public final class ProAVPlayer { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - innerPlayer.shutdown() + atmosOutput = nil + updateInnerPlayerItem { innerPlayer.shutdown() } remuxItem?.shutdown() session?.requestCleanup() server.stop() @@ -167,24 +310,85 @@ public final class ProAVPlayer { delegate?.finish(player: self, error: error) } - private func restart(at time: TimeInterval, completion: ((Bool) -> Void)?) { + private func restart(at time: TimeInterval, completion: ((Bool) -> Void)?, spatialAudioEnabled: Bool = true) { abortPendingSourceSwitch() + // Keep the tracks the launch's probe refused. + if let session { + refusedSpatialAudioTrackIDs.formUnion(session.spatialAudioPolicy.refusedTrackIDs) + } let previousCompletion = pendingSeekCompletion pendingSeekCompletion = nil previousCompletion?(false) pendingSeekCompletion = completion + fallbackResumesPlayback = nil didFail = false remuxItem?.delegate = nil - innerPlayer.shutdown() + atmosOutput = nil + updateInnerPlayerItem { innerPlayer.shutdown() } remuxItem?.shutdown() session?.requestCleanup() - startRemux(at: time) + startRemux(at: time, spatialAudioEnabled: spatialAudioEnabled) + } + + private func startSourceSwitch(url: URL, options: KSOptions, spatialAudio: ProAVRemuxSession.SpatialAudioPolicy, completion: @escaping (Bool) -> Void) { + let time = currentPlaybackTime + let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: nil, spatialAudio: spatialAudio) + pendingSourceSwitch = PendingSourceSwitch(launch: launch, url: url, options: options, startOffset: time, spatialAudio: spatialAudio, completion: completion) + launch.session.onReady = { [weak self, weak session = launch.session] _ in + runOnMainThread { + guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } + self.pendingSourceSwitchDidBecomeReady() + } + } + // One handler serves the candidate while it is pending and once it is committed, so a failure + // that arrives during the commit still reaches the active session. + launch.session.onFailure = { [weak self, weak session = launch.session] sessionError in + runOnMainThread { + guard let self, let session else { return } + if self.pendingSourceSwitch?.launch.session === session { + self.pendingSourceSwitchDidFail(session) + } else if self.session === session { + self.sessionDidFail(session, error: sessionError) + } + } + } + launch.item.prepareToPlay() + } + + /// Retries a candidate whose APAC conversion failed on the FLAC path, as a cold launch would. + private func pendingSourceSwitchDidFail(_ session: ProAVRemuxSession) { + guard let pending = pendingSourceSwitch, let failure = session.spatialAudioFailure else { + abortPendingSourceSwitch(restoreSpatialAudio: true) + return + } + var spatialAudio = pending.spatialAudio + spatialAudio.isEnabled = false + if case let .refused(trackID) = failure { + spatialAudio.refusedTrackIDs.insert(trackID) + } + restartPendingSourceSwitch(spatialAudio: spatialAudio) + } + + private func restartPendingSourceSwitch(spatialAudio: ProAVRemuxSession.SpatialAudioPolicy) { + guard let pending = pendingSourceSwitch else { return } + pendingSourceSwitch = nil + if pending.innerSwitchStarted { + innerPlayer.abandonPendingSourceSwitch() + } + pending.launch.item.delegate = nil + pending.launch.item.shutdown() + pending.launch.session.requestCleanup() + startSourceSwitch(url: pending.url, options: pending.options, spatialAudio: spatialAudio, completion: pending.completion) } private func pendingSourceSwitchDidBecomeReady() { guard let pending = pendingSourceSwitch else { return } + if pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: externalPlaybackActive) { + restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) + return + } guard let serverBaseURL else { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) return } let masterURL = serverBaseURL.appendingPathComponent(pending.launch.directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) @@ -195,7 +399,7 @@ public final class ProAVPlayer { if success { self.commitPendingSourceSwitch() } else { - self.abortPendingSourceSwitch() + self.abortPendingSourceSwitch(restoreSpatialAudio: true) } } } @@ -211,27 +415,23 @@ public final class ProAVPlayer { url = pending.url options = pending.options preferredAudioTrackID = nil + refusedSpatialAudioTrackIDs = pending.launch.session.spatialAudioPolicy.refusedTrackIDs session = pending.launch.session remuxItem = pending.launch.item delegateProxy = pending.launch.proxy activeLaunchIndex = pending.launch.index startOffset = pending.startOffset pending.launch.session.onReady = nil - pending.launch.session.onFailure = { [weak self, weak session = pending.launch.session] sessionError in - runOnMainThread { - guard let self, let session, self.session === session else { return } - self.fail(error: sessionError) - } - } previousItem?.shutdown() previousSession?.requestCleanup() if session?.videoSignaling?.preferredDynamicRange != previousDynamicRange { applyDisplayCriteria() } + updateSpatialAudioOutput() pending.completion(true) } - private func abortPendingSourceSwitch() { + private func abortPendingSourceSwitch(restoreSpatialAudio: Bool = false) { guard let pending = pendingSourceSwitch else { return } pendingSourceSwitch = nil if pending.innerSwitchStarted { @@ -241,11 +441,15 @@ public final class ProAVPlayer { pending.launch.item.shutdown() pending.launch.session.requestCleanup() pending.completion(false) + // AirPlay may have ended while this candidate was pending, deferring the active restart. + if restoreSpatialAudio, pendingSourceSwitch == nil, !externalPlaybackActive { + _ = restartForExternalPlaybackIfNeeded() + } } fileprivate func launchDidFail(index: Int, error: NSError?) { if let pending = pendingSourceSwitch, pending.launch.index == index { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) return } guard index == activeLaunchIndex else { return } @@ -308,6 +512,8 @@ extension ProAVPlayer: MediaPlayerProtocol { public var pipController: KSPictureInPictureController? { innerPlayer.pipController } public func prepareToPlay() { + // A fresh play must not inherit the route from a previous, terminal shutdown. + externalPlaybackActive = innerPlayer.isExternalPlaybackActive didFail = false startRemux(at: options.startPlayTime) } @@ -317,15 +523,22 @@ extension ProAVPlayer: MediaPlayerProtocol { didFail = false reportedReady = false preferredAudioTrackID = nil + refusedSpatialAudioTrackIDs = [] self.url = url self.options = options } public func play() { + if fallbackResumesPlayback != nil { + fallbackResumesPlayback = true + } innerPlayer.play() } public func pause() { + if fallbackResumesPlayback != nil { + fallbackResumesPlayback = false + } innerPlayer.pause() } @@ -335,26 +548,11 @@ extension ProAVPlayer: MediaPlayerProtocol { completion(false) return } - let time = currentPlaybackTime - let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: nil) - pendingSourceSwitch = PendingSourceSwitch(launch: launch, url: url, options: options, startOffset: time, completion: completion) - launch.session.onReady = { [weak self, weak session = launch.session] _ in - runOnMainThread { - guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } - self.pendingSourceSwitchDidBecomeReady() - } - } - launch.session.onFailure = { [weak self, weak session = launch.session] _ in - runOnMainThread { - guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } - self.abortPendingSourceSwitch() - } - } - launch.item.prepareToPlay() + startSourceSwitch(url: url, options: options, spatialAudio: ProAVRemuxSession.SpatialAudioPolicy(), completion: completion) } public func cancelSourceSwitch() { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) } public func shutdown() { @@ -363,13 +561,15 @@ extension ProAVPlayer: MediaPlayerProtocol { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - innerPlayer.shutdown() + atmosOutput = nil + updateInnerPlayerItem { innerPlayer.shutdown() } remuxItem?.shutdown() session?.requestCleanup() remuxItem = nil session = nil server.stop() serverBaseURL = nil + externalPlaybackActive = false } public func seek(time: TimeInterval, completion: @escaping ((Bool) -> Void)) { @@ -410,7 +610,14 @@ extension ProAVPlayer: MediaPlayerDelegate { pendingSeekCompletion = nil if reportedReady { seekCompletion?(true) - if options.isSeekedAutoPlay { + if let resumesPlayback = fallbackResumesPlayback { + fallbackResumesPlayback = nil + if resumesPlayback { + play() + } else { + pause() + } + } else if options.isSeekedAutoPlay { play() } } else { diff --git a/Sources/Lumen/MEPlayer/ProAVPlaylist.swift b/Sources/Lumen/MEPlayer/ProAVPlaylist.swift index 6925798..3f97581 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlaylist.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlaylist.swift @@ -127,6 +127,7 @@ struct ProAVVideoSignaling { struct ProAVAudioSignaling: Equatable { let codecsAttribute: String let channels: String? + var mediaPlaylistName: String? = nil } enum ProAVAudioStrategy { @@ -203,6 +204,9 @@ enum ProAVPlaylist { if let channels = audio.channels { mediaAttributes.append("CHANNELS=\"\(channels)\"") } + if let mediaPlaylistName = audio.mediaPlaylistName { + mediaAttributes.append("URI=\"\(mediaPlaylistName)\"") + } lines.append("#EXT-X-MEDIA:" + mediaAttributes.joined(separator: ",")) } lines.append("#EXT-X-STREAM-INF:" + attributes.joined(separator: ",")) diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 4295250..1b3ac1a 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -17,11 +17,37 @@ public final class ProAVRemuxSession: @unchecked Sendable { static let masterPlaylistName = "master.m3u8" static let mediaPlaylistName = "media.m3u8" static let initSegmentName = "init.mp4" + static let audioPlaylistName = "audio.m3u8" + static let audioInitSegmentName = "audio-init.mp4" + + /// Which audio tracks of a launch try the APAC conversion before FLAC. + struct SpatialAudioPolicy: Equatable { + /// Tracks the library refused, which keep the FLAC path for their source. + var refusedTrackIDs = Set() + /// Off for a launch that retries on FLAC after the conversion failed without refusing the track. + var isEnabled = true + var isExternalPlaybackActive = false + + func allows(trackID: Int32) -> Bool { + isEnabled && !isExternalPlaybackActive && !refusedTrackIDs.contains(trackID) + } + } + + enum SpatialAudioFailure: Equatable { + /// The library refused the track, as it does for TrueHD without Atmos or DTS-HD MA without DTS:X. + case refused(trackID: Int32) + /// The conversion failed for this launch only. + case failed + } let configuration: Configuration private let lock = NSLock() private var pendingEvents = [() -> Void]() private var _preferredAudioTrackID: Int32? + private var _spatialAudioPolicy = SpatialAudioPolicy() + private var _spatialAudioFailure: SpatialAudioFailure? + private var _spatialAudioWithheld = false + private var _playbackTime: TimeInterval? private var _onReady: ((URL) -> Void)? private var _onFailure: ((NSError) -> Void)? private var _videoSignaling: ProAVVideoSignaling? @@ -33,9 +59,12 @@ public final class ProAVRemuxSession: @unchecked Sendable { private var initScanner = ProAVInitBoundaryScanner() private var initBoundaryFound = false private var segments = [ProAVSegment]() + private var audioSegments = [ProAVSegment]() private var segmentIndex = 0 private var segmentStart: Double? private var lastVideoSeconds: Double? + private var lastAudioSeconds: Double? + private var audioEndSeconds: Double? private var currentSegmentBytes = Int64(0) private var readyFired = false private var finished = false @@ -64,6 +93,71 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + var spatialAudioPolicy: SpatialAudioPolicy { + get { + withLock { _spatialAudioPolicy } + } + set { + withLock { _spatialAudioPolicy = newValue } + } + } + + /// Whether the audio plays as the separate APAC rendition. + var playsSpatialAudio: Bool { + withLock { audioSignaling?.mediaPlaylistName != nil } + } + + /// Set when the session failed because the APAC conversion did; the FLAC path can still play the track. + var spatialAudioFailure: SpatialAudioFailure? { + withLock { _spatialAudioFailure } + } + + /// Whether this launch left APAC out for an eligible track that its source has not refused, as during + /// AirPlay, so a later launch can bring it back. + var spatialAudioWithheld: Bool { + get { + withLock { _spatialAudioWithheld } + } + set { + withLock { _spatialAudioWithheld = newValue } + } + } + + /// Recheck after opening as well as on route changes: the route can change before audio is selected. + func needsSpatialAudioRestart(isExternalPlaybackActive: Bool) -> Bool { + withLock { + isExternalPlaybackActive ? audioSignaling?.mediaPlaylistName != nil : _spatialAudioWithheld + } + } + + /// The player's position from the start of the file, which the remux reads ahead of. + var playbackTime: TimeInterval? { + get { + withLock { _playbackTime } + } + set { + withLock { _playbackTime = newValue } + } + } + + /// Pace the least advanced required stream so pausing the only reader cannot starve audio. + /// Source timestamps start at `sourceStart`; playback counts from the start of the file. + func remuxedAhead(sourceStart: TimeInterval) -> TimeInterval? { + withLock { + guard readyFired, let lastVideoSeconds, let _playbackTime else { return nil } + if audioSignaling != nil { + guard let lastAudioSeconds else { return nil } + // Once all declared audio is read, its shorter duration must not make the video + // tail read without a bound. Unknown ends still wait for audio to avoid starvation. + if let audioEndSeconds, lastAudioSeconds >= audioEndSeconds - 0.000001 { + return lastVideoSeconds - sourceStart - _playbackTime + } + return min(lastVideoSeconds, lastAudioSeconds) - sourceStart - _playbackTime + } + return lastVideoSeconds - sourceStart - _playbackTime + } + } + var onReady: ((URL) -> Void)? { get { withLock { _onReady } @@ -149,6 +243,14 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + func trackAudioTime(seconds: Double, duration: Double = 0, streamEnd: Double? = nil) { + withLock { + let end = seconds + max(duration, 0) + lastAudioSeconds = max(lastAudioSeconds ?? end, end) + audioEndSeconds = streamEnd + } + } + func closeSegment(nextStartTime: Double) { withLock { guard let start = segmentStart else { @@ -159,7 +261,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { segmentStart = nextStartTime currentHandle = openFileLocked(named: segmentFileName(index: segmentIndex)) currentSegmentBytes = 0 - writeMediaPlaylistLocked(ended: false) + writePlaylistsLocked(ended: false) fireReadyIfNeededLocked(force: false) } } @@ -181,8 +283,12 @@ public final class ProAVRemuxSession: @unchecked Sendable { if reachedEnd, !failed { if segments.isEmpty { failLocked(NSError(description: "ProAV remux produced no segments")) + } else if audioSignaling?.mediaPlaylistName != nil, audioSegments.isEmpty { + // The session could never become ready; the FLAC transcode can still play the track. + _spatialAudioFailure = .failed + failLocked(NSError(description: "ProAV spatial audio produced no fragments")) } else { - writeMediaPlaylistLocked(ended: true) + writePlaylistsLocked(ended: true) fireReadyIfNeededLocked(force: true) } } @@ -198,6 +304,33 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { failLocked(error) } } + func failSpatialAudio(_ error: NSError, failure: SpatialAudioFailure) { + withLock { + guard !failed else { return } + _spatialAudioFailure = failure + failLocked(error) + } + } + + func writeAudioInitSegment(_ data: Data) { + withLock { + guard !finished, !failed else { return } + writeLocked(data: data, named: Self.audioInitSegmentName) + } + } + + func appendAudioSegment(_ data: Data, duration: Double) { + withLock { + guard !finished, !failed else { return } + let fileName = "audio\(audioSegments.count).m4s" + writeLocked(data: data, named: fileName) + guard !failed else { return } + audioSegments.append(ProAVSegment(fileName: fileName, duration: duration)) + writePlaylistsLocked(ended: false) + fireReadyIfNeededLocked(force: false) + } + } + func requestCleanup() { withLock { cleanupWhenFinished = true @@ -252,14 +385,22 @@ public final class ProAVRemuxSession: @unchecked Sendable { segmentIndex += 1 } - private func writeMediaPlaylistLocked(ended: Bool) { - let media = ProAVPlaylist.media(targetDuration: configuration.targetSegmentDuration, initSegmentName: Self.initSegmentName, segments: segments, ended: ended) + /// Writes the video playlist and the separate audio rendition's, which HLS requires to share one + /// target duration, so a longer segment in either raises both. + private func writePlaylistsLocked(ended: Bool) { + let targetDuration = max(configuration.targetSegmentDuration, (segments + audioSegments).map(\.duration).max() ?? 0) + let media = ProAVPlaylist.media(targetDuration: targetDuration, initSegmentName: Self.initSegmentName, segments: segments, ended: ended) writeLocked(text: media, to: mediaURL) + guard audioSignaling?.mediaPlaylistName != nil else { return } + let audio = ProAVPlaylist.media(targetDuration: targetDuration, initSegmentName: Self.audioInitSegmentName, segments: audioSegments, ended: ended) + writeLocked(text: audio, to: configuration.directory.appendingPathComponent(Self.audioPlaylistName)) } private func fireReadyIfNeededLocked(force: Bool) { guard !readyFired, !failed, !segments.isEmpty else { return } guard force || segments.count >= configuration.minimumSegmentsBeforeReady else { return } + // A separate audio rendition needs a fragment too before the player can start. + guard audioSignaling?.mediaPlaylistName == nil || !audioSegments.isEmpty else { return } readyFired = true if let handler = _onReady { let url = masterURL @@ -283,6 +424,14 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + private func writeLocked(data: Data, named name: String) { + do { + try data.write(to: configuration.directory.appendingPathComponent(name)) + } catch { + failLocked(NSError(description: "ProAV segment write failed: \(name)")) + } + } + private func write(buffer: UnsafePointer?, size: Int32) -> Int32 { withLock { guard let buffer, size > 0 else { return size } diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift new file mode 100644 index 0000000..4758949 --- /dev/null +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -0,0 +1,275 @@ +import CoreMedia +import Foundation +import Libavcodec +import Libavformat +#if canImport(SpatialAudioBridge) +import SpatialAudioBridge +#endif + +/// Converts a TrueHD Atmos or DTS:X track to APAC, which the remux serves as a separate audio +/// rendition instead of the FLAC transcode. A probe before the launch keeps a track the library refuses +/// on FLAC; one the probe cannot judge fails the session as decoding starts, and the player then +/// plays it through the FLAC transcode. +/// +/// The read thread hands packets over and blocks while the conversion is behind, so the remux runs at +/// its pace. Fragments are placed on the muxer's timeline once `begin(origin:videoStart:)` says where +/// the video landed; packets that arrive before then are held. +final class ProAVSpatialAudioTranscoder: @unchecked Sendable { + private enum Input { + case timeline(origin: CMTime, videoStart: CMTime) + case packet(Data, time: CMTime?) + } + + let streamIndex: Int + let signaling: ProAVAudioSignaling + private let timebase: Timebase + private let queue: AsyncStream.Continuation + private let capacity = DispatchSemaphore(value: 64) + private let completion = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var cancelled = false + private(set) var hasTimeline = false + + /// Returns nil below tvOS, iOS and macOS 26 and for any track other than 48 kHz TrueHD or DTS-HD + /// Master Audio, which keep the FLAC path. + init?(stream: UnsafeMutablePointer, session: ProAVRemuxSession) { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *), + let codecpar = stream.pointee.codecpar, + let codec = Self.codec(codecpar: codecpar.pointee) + else { + return nil + } + // The offline profile would hold the first fragment back for about 20 s of audio. + let configuration = APACConfiguration(liveEncoding: true) + guard let priming = try? configuration.encoderPrimingDuration() else { + return nil + } + streamIndex = Int(stream.pointee.index) + timebase = Timebase(stream.pointee.time_base) + signaling = ProAVAudioSignaling(codecsAttribute: configuration.layout.apacCodecString, channels: "\(configuration.layout.channelCount)", mediaPlaylistName: ProAVRemuxSession.audioPlaylistName) + let (inputs, queue) = AsyncStream.makeStream(of: Input.self) + self.queue = queue + Task.detached { [self] in + await convert(inputs, codec: codec, configuration: configuration, priming: priming, session: session) + } + #else + return nil + #endif + } + + private var isCancelled: Bool { + lock.lock() + defer { lock.unlock() } + return cancelled + } + + /// Places the audio on the muxer's timeline, which starts at `origin`, the first video decode + /// time, and starts it with the first video frame at `videoStart`. + func begin(origin: CMTime, videoStart: CMTime) { + hasTimeline = true + capacity.wait() + queue.yield(.timeline(origin: origin, videoStart: videoStart)) + } + + /// The muxer's timeline for its first video packet: the muxer shifts it so that the packet decodes + /// at zero, deriving a missing decode time from the reorder delay as it does. Nil without any time. + static func timeline(pts: Int64, dts: Int64, duration: Int64, videoDelay: Int32, timebase: Timebase) -> (origin: CMTime, videoStart: CMTime)? { + let start = pts == Int64.min ? dts : pts + guard start != Int64.min else { + return nil + } + let origin = dts == Int64.min ? start - Int64(videoDelay) * duration : dts + return (timebase.cmtime(for: origin), timebase.cmtime(for: start)) + } + + func append(packet: UnsafeMutablePointer) { + guard let data = packet.pointee.data, packet.pointee.size > 0 else { return } + let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts + let time = timestamp == Int64.min ? nil : timebase.cmtime(for: timestamp) + capacity.wait() + queue.yield(.packet(Data(bytes: data, count: Int(packet.pointee.size)), time: time)) + } + + /// Converts the rest and waits until the last fragment reached the session. + func finish() { + queue.finish() + completion.wait() + } + + func cancel() { + lock.lock() + cancelled = true + lock.unlock() + queue.finish() + } + + /// Whether a track would convert on this system, given the chance. + static func converts(codecpar: AVCodecParameters) -> Bool { + #if canImport(SpatialAudioBridge) + if #available(macOS 26, iOS 26, tvOS 26, *) { + return codec(codecpar: codecpar) != nil + } + #endif + return false + } + + /// Why the library will never render the track, found by decoding about a second of it from its + /// first sync point without encoding; nil when it renders or the probe cannot tell. Reads the + /// demuxer onward from where it is. + static func probeRefusal(stream: UnsafeMutablePointer, formatContext: UnsafeMutablePointer) -> String? { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *), + let codecpar = stream.pointee.codecpar, + let codec = codec(codecpar: codecpar.pointee) + else { + return nil + } + var packet = av_packet_alloc() + defer { av_packet_free(&packet) } + guard let packet else { return nil } + let timebase = Timebase(stream.pointee.time_base) + var audio = Data() + var start: CMTime? + var bytesRead = 0 + // The video read around the audio counts as well, so a track without a sync point ends it. + while bytesRead < 32 << 20, audio.count < 8 << 20, av_read_frame(formatContext, packet) == 0 { + defer { av_packet_unref(packet) } + bytesRead += Int(packet.pointee.size) + guard packet.pointee.stream_index == stream.pointee.index, let bytes = packet.pointee.data, packet.pointee.size > 0 else { continue } + let data = Data(bytes: bytes, count: Int(packet.pointee.size)) + guard !audio.isEmpty || codec.isSyncPoint(data) else { continue } + let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts + if timestamp != Int64.min { + let time = timebase.cmtime(for: timestamp) + if let start, (time - start).seconds >= 1 { + break + } + start = start ?? time + } + audio += data + } + // Damaged audio is for the conversion to judge, which falls back for this launch only. + guard case let .unsupported(reason)? = try? codec.probe(audio) else { + return nil + } + return reason + #else + return nil + #endif + } + + #if canImport(SpatialAudioBridge) + @available(macOS 26, iOS 26, tvOS 26, *) + static func codec(codecpar: AVCodecParameters) -> SpatialAudioCodec? { + guard codecpar.sample_rate == 48000 else { + return nil + } + // FFmpeg reports some DTS:X presentations as plain DTS-HD MA, so every one is offered and the + // library tells them apart by decoding. TrueHD goes by FFmpeg's Atmos label. + switch codecpar.codec_id { + case AV_CODEC_ID_TRUEHD where codecpar.profile == AV_PROFILE_TRUEHD_ATMOS: + return .trueHDAtmos + case AV_CODEC_ID_DTS where [AV_PROFILE_DTS_HD_MA, AV_PROFILE_DTS_HD_MA_X, AV_PROFILE_DTS_HD_MA_X_IMAX].contains(codecpar.profile): + return .dtsX + default: + return nil + } + } + + @available(macOS 26, iOS 26, tvOS 26, *) + private func convert(_ inputs: AsyncStream, codec: SpatialAudioCodec, configuration: APACConfiguration, priming: CMTime, session: ProAVRemuxSession) async { + var transcoder: SpatialAudioTranscoder? + var timeline: (origin: CMTime, start: CMTime)? + var held = [(data: Data, time: CMTime?)]() + var unstartedBytes = 0 + var failed = false + // Any error ends the session at once, so the player restarts on the FLAC path; an unsupported + // track stays there. + func fail(_ error: Error) { + failed = true + guard !isCancelled else { return } + session.failSpatialAudio(NSError(description: "ProAV spatial audio failed: \(error)"), failure: Self.failure(for: error, trackID: Int32(streamIndex))) + } + func append(_ data: Data, time: CMTime?) async throws { + guard let timeline else { + held.append((data, time)) + return + } + if transcoder == nil { + // Decoding starts at a TrueHD major sync or a DTS core sync; earlier audio is discarded + // and a later start is padded with silence, so the audio begins with the video. + guard let time, codec.isSyncPoint(data) else { return } + var settings = configuration + settings.startTime = timeline.start + settings.inputStartTime = time - timeline.origin + transcoder = try SpatialAudioTranscoder(codec: codec, destination: .segments { segment in + switch segment.kind { + case .initialization: + session.writeAudioInitSegment(segment.data) + case .media: + guard let duration = segment.duration?.seconds, duration > 0 else { + throw SpatialAudioError.encoding("APAC fragment without a duration") + } + session.appendAudioSegment(segment.data, duration: duration) + } + }, configuration: settings) + } + try await transcoder?.append(data, time: time.map { $0 - timeline.origin }) + } + // A failed or cancelled conversion keeps draining, so the read thread never waits on it. + for await input in inputs { + defer { capacity.signal() } + guard !failed, !isCancelled else { continue } + do { + switch input { + case let .timeline(origin, videoStart): + // Fragments keep the encoder priming before the start, which cannot go below zero. + timeline = (origin, CMTimeMaximum(videoStart - origin, priming)) + let waiting = held + held = [] + for packet in waiting { + try await append(packet.data, time: packet.time) + } + case let .packet(data, time): + // Audio held for the timeline or skipped to a sync point is bounded, so a stream that + // never starts the conversion falls back to FLAC instead of buffering the file. + if transcoder == nil { + unstartedBytes += data.count + guard unstartedBytes <= 8 << 20 else { + throw SpatialAudioError.source("No sync point or video timeline within 8 MB of audio") + } + } + try await append(data, time: time) + } + } catch { + fail(error) + } + } + if isCancelled { + await transcoder?.cancel() + } else if !failed { + do { + guard let transcoder else { + throw SpatialAudioError.source("The track has no sync point to start from") + } + _ = try await transcoder.finish() + } catch { + fail(error) + } + } + completion.signal() + } + + /// How the player responds: an unsupported track keeps FLAC for its source, and anything else + /// falls back for this launch only. That includes a discontinuity: its point lies ahead of + /// playback, so restarting the launch would meet it again, and the next seek starts a new session. + @available(macOS 26, iOS 26, tvOS 26, *) + static func failure(for error: Error, trackID: Int32) -> ProAVRemuxSession.SpatialAudioFailure { + if case .unsupported? = error as? SpatialAudioError { + return .refused(trackID: trackID) + } + return .failed + } + #endif +} diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift new file mode 100644 index 0000000..2bae8da --- /dev/null +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -0,0 +1,307 @@ +import CoreMedia +import FFmpegKit +import Libavcodec +import Libavformat +@testable import Lumen +#if canImport(SpatialAudioBridge) +import SpatialAudioBridge +#endif +import XCTest + +class ProAVSpatialAudioTest: XCTestCase { + private let apac = ProAVAudioSignaling(codecsAttribute: "apac.31.03", channels: "12", mediaPlaylistName: ProAVRemuxSession.audioPlaylistName) + private let video = ProAVVideoSignaling(codecTag: "avc1", codecsAttribute: "avc1.640028", videoRange: "SDR", supplementalCodecs: nil, preferredDynamicRange: .sdr) + + private func makeSession() -> ProAVRemuxSession { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + return ProAVRemuxSession(configuration: .init(directory: directory)) + } + + func testMasterPointsAtTheAPACRendition() { + let video = ProAVVideoSignaling(codecTag: "dvh1", codecsAttribute: "dvh1.08.06", videoRange: "PQ", supplementalCodecs: nil, preferredDynamicRange: .dolbyVision) + let master = ProAVPlaylist.master(mediaPlaylistName: "media.m3u8", video: video, audio: apac, bandwidth: 24_000_000, resolution: CGSize(width: 3840, height: 2160), frameRate: 23.976) + XCTAssertTrue(master.contains("#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"main\",NAME=\"Original\",DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"12\",URI=\"audio.m3u8\"\n")) + XCTAssertTrue(master.contains("CODECS=\"dvh1.08.06,apac.31.03\"")) + XCTAssertTrue(master.contains("AUDIO=\"main\"")) + } + + func testOnlyAtmosTrueHDAndLosslessDTSAreOffered() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + func codec(_ codecID: AVCodecID, profile: Int32, sampleRate: Int32 = 48000) -> SpatialAudioCodec? { + var codecpar = AVCodecParameters() + codecpar.codec_id = codecID + codecpar.profile = profile + codecpar.sample_rate = sampleRate + return ProAVSpatialAudioTranscoder.codec(codecpar: codecpar) + } + XCTAssertEqual(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_TRUEHD_ATMOS), .trueHDAtmos) + XCTAssertNil(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_UNKNOWN)) + for profile in [AV_PROFILE_DTS_HD_MA, AV_PROFILE_DTS_HD_MA_X, AV_PROFILE_DTS_HD_MA_X_IMAX] { + XCTAssertEqual(codec(AV_CODEC_ID_DTS, profile: profile), .dtsX) + } + for profile in [AV_PROFILE_DTS, AV_PROFILE_DTS_HD_HRA, AV_PROFILE_DTS_EXPRESS, AV_PROFILE_UNKNOWN] { + XCTAssertNil(codec(AV_CODEC_ID_DTS, profile: profile)) + } + XCTAssertNil(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_TRUEHD_ATMOS, sampleRate: 96000)) + XCTAssertNil(codec(AV_CODEC_ID_EAC3, profile: AV_PROFILE_EAC3_DDP_ATMOS)) + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + + func testOnlyUnsupportedTracksAreRefused() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + XCTAssertEqual(ProAVSpatialAudioTranscoder.failure(for: SpatialAudioError.unsupported("This TrueHD stream has no Atmos presentation"), trackID: 1), .refused(trackID: 1)) + // A discontinuity lies ahead of playback, so like a transient error it falls back for the launch. + let launchOnly: [Error] = [ + SpatialAudioError.discontinuity("Timestamps jumped"), + SpatialAudioError.decoding("Atmos presentation has no object metadata within 0.5 s"), + SpatialAudioError.encoding("APAC encoder stopped"), + SpatialAudioError.source("No sync point or video timeline within 8 MB of audio"), + SpatialAudioError.coreMedia(-12780), + CancellationError(), + ] + for error in launchOnly { + XCTAssertEqual(ProAVSpatialAudioTranscoder.failure(for: error, trackID: 1), .failed, "\(error)") + } + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + + func testTimelineFollowsTheMuxerShift() { + let timebase = Timebase(num: 1, den: 1000) + let demuxed = ProAVSpatialAudioTranscoder.timeline(pts: 1080, dts: 1000, duration: 40, videoDelay: 2, timebase: timebase) + XCTAssertEqual(demuxed?.origin, CMTime(value: 1000, timescale: 1000)) + XCTAssertEqual(demuxed?.videoStart, CMTime(value: 1080, timescale: 1000)) + // The first packet after a seek has no decode time, so the muxer derives it from the B-frame delay. + let afterSeek = ProAVSpatialAudioTranscoder.timeline(pts: 1000, dts: Int64.min, duration: 40, videoDelay: 2, timebase: timebase) + XCTAssertEqual(afterSeek?.origin, CMTime(value: 920, timescale: 1000)) + XCTAssertEqual(afterSeek?.videoStart, CMTime(value: 1000, timescale: 1000)) + let withoutReordering = ProAVSpatialAudioTranscoder.timeline(pts: 1000, dts: Int64.min, duration: 40, videoDelay: 0, timebase: timebase) + XCTAssertEqual(withoutReordering?.origin, CMTime(value: 1000, timescale: 1000)) + let decodeTimeOnly = ProAVSpatialAudioTranscoder.timeline(pts: Int64.min, dts: 500, duration: 40, videoDelay: 0, timebase: timebase) + XCTAssertEqual(decodeTimeOnly?.origin, CMTime(value: 500, timescale: 1000)) + XCTAssertEqual(decodeTimeOnly?.videoStart, CMTime(value: 500, timescale: 1000)) + XCTAssertNil(ProAVSpatialAudioTranscoder.timeline(pts: Int64.min, dts: Int64.min, duration: 40, videoDelay: 2, timebase: timebase)) + } + + func testConversionThatNeverStartsFallsBackForTheLaunch() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + let formatContext = avformat_alloc_context() + defer { avformat_free_context(formatContext) } + let stream = try XCTUnwrap(avformat_new_stream(formatContext, nil)) + stream.pointee.codecpar.pointee.codec_type = AVMEDIA_TYPE_AUDIO + stream.pointee.codecpar.pointee.codec_id = AV_CODEC_ID_TRUEHD + stream.pointee.codecpar.pointee.profile = AV_PROFILE_TRUEHD_ATMOS + stream.pointee.codecpar.pointee.sample_rate = 48000 + stream.pointee.time_base = AVRational(num: 1, den: 1000) + let session = makeSession() + let transcoder = try XCTUnwrap(ProAVSpatialAudioTranscoder(stream: stream, session: session)) + defer { transcoder.cancel() } + transcoder.begin(origin: .zero, videoStart: .zero) + var packet = av_packet_alloc() + defer { av_packet_free(&packet) } + let corePacket = try XCTUnwrap(packet) + XCTAssertEqual(av_new_packet(corePacket, 64 * 1024), 0) + memset(corePacket.pointee.data, 0, 64 * 1024) + // 130 packets of 64 KB without a TrueHD major sync pass the 8 MB bound. + for index in 0 ..< 130 { + corePacket.pointee.pts = Int64(index) * 50 + transcoder.append(packet: corePacket) + } + let deadline = Date(timeIntervalSinceNow: 10) + while !session.isFailed, Date() < deadline { + RunLoop.current.run(until: Date(timeIntervalSinceNow: 0.01)) + } + XCTAssertEqual(session.spatialAudioFailure, .failed) + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + + func testReadyWaitsForTheAPACRendition() throws { + let session = makeSession() + var readyCount = 0 + session.onReady = { _ in readyCount += 1 } + XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + XCTAssertTrue(session.playsSpatialAudio) + for start in [0.0, 2, 4] { + session.closeSegment(nextStartTime: start) + } + XCTAssertEqual(readyCount, 0) + session.writeAudioInitSegment(Data([0])) + session.appendAudioSegment(Data([1]), duration: 3.2) + XCTAssertEqual(readyCount, 1) + session.finish(reachedEnd: true) + let directory = session.configuration.directory + let playlist = try String(contentsOf: directory.appendingPathComponent(ProAVRemuxSession.audioPlaylistName), encoding: .utf8) + XCTAssertTrue(playlist.contains("#EXT-X-MAP:URI=\"audio-init.mp4\"\n#EXTINF:3.20000,\naudio0.m4s\n#EXT-X-ENDLIST\n")) + // HLS requires the audio and video playlists of a variant to share one target duration. + let media = try String(contentsOf: directory.appendingPathComponent(ProAVRemuxSession.mediaPlaylistName), encoding: .utf8) + XCTAssertTrue(playlist.contains("#EXT-X-TARGETDURATION:4\n")) + XCTAssertTrue(media.contains("#EXT-X-TARGETDURATION:4\n")) + } + + func testEndingWithoutAPACFragmentsFallsBackForTheLaunch() { + let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.closeSegment(nextStartTime: 0) + session.closeSegment(nextStartTime: 2) + session.finish(reachedEnd: true) + XCTAssertTrue(session.isFailed) + XCTAssertEqual(session.spatialAudioFailure, .failed) + } + + func testOnlySpatialAudioFailuresFallBackToFLAC() { + let failed = makeSession() + failed.fail(NSError(description: "segment write failed")) + failed.failSpatialAudio(NSError(description: "unsupported metadata"), failure: .refused(trackID: 1)) + XCTAssertTrue(failed.isFailed) + XCTAssertNil(failed.spatialAudioFailure) + let refused = makeSession() + refused.failSpatialAudio(NSError(description: "unsupported metadata"), failure: .refused(trackID: 1)) + XCTAssertTrue(refused.isFailed) + XCTAssertEqual(refused.spatialAudioFailure, .refused(trackID: 1)) + } + + func testRefusedTracksKeepFLACWhileOthersTryAPAC() { + var policy = ProAVRemuxSession.SpatialAudioPolicy(refusedTrackIDs: [1]) + XCTAssertFalse(policy.allows(trackID: 1)) + XCTAssertTrue(policy.allows(trackID: 2)) + policy.isEnabled = false + XCTAssertFalse(policy.allows(trackID: 2)) + } + + func testAirPlayPreservesConversionIntentAndRefusedTracks() { + var policy = ProAVRemuxSession.SpatialAudioPolicy(refusedTrackIDs: [1]) + policy.isExternalPlaybackActive = true + XCTAssertFalse(policy.allows(trackID: 2)) + policy.isExternalPlaybackActive = false + XCTAssertTrue(policy.allows(trackID: 2)) + XCTAssertFalse(policy.allows(trackID: 1)) + policy.isEnabled = false + policy.isExternalPlaybackActive = true + policy.isExternalPlaybackActive = false + XCTAssertFalse(policy.allows(trackID: 2)) + } + + func testOpeningLaunchRechecksAirPlayAfterAudioIsSelected() { + let session = makeSession() + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: true)) + XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + XCTAssertTrue(session.needsSpatialAudioRestart(isExternalPlaybackActive: true)) + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: false)) + session.finish(reachedEnd: false) + } + + func testAirPlayWithheldLaunchRetriesOnTheLocalRoute() { + let session = makeSession() + let flac = ProAVAudioSignaling(codecsAttribute: "fLaC", channels: "8") + XCTAssertTrue(session.begin(signaling: video, audioSignaling: flac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: false)) + session.spatialAudioWithheld = true + XCTAssertTrue(session.needsSpatialAudioRestart(isExternalPlaybackActive: false)) + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: true)) + session.finish(reachedEnd: false) + } + + func testReadAheadWaitsForBothVideoAndAudio() { + let flac = ProAVAudioSignaling(codecsAttribute: "fLaC", channels: "8") + for audio in [apac, flac] { + let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: audio, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.playbackTime = 600 + session.trackVideoTime(seconds: 641.4) + for start in [600.0, 602, 604] { + session.closeSegment(nextStartTime: start) + } + if audio.mediaPlaylistName != nil { + session.writeAudioInitSegment(Data([0])) + session.appendAudioSegment(Data([1]), duration: 3.2) + } + XCTAssertNil(session.remuxedAhead(sourceStart: 1.4)) + session.trackAudioTime(seconds: 603.4) + // Forty seconds of video must not pause the reader with only two seconds of audio. + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 2, accuracy: 0.001) + session.trackAudioTime(seconds: 651.4) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 40, accuracy: 0.001) + session.trackAudioTime(seconds: 602) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 40, accuracy: 0.001) + session.finish(reachedEnd: false) + } + } + + func testReadAheadUsesVideoAfterTheDeclaredAudioEnd() { + let flac = ProAVAudioSignaling(codecsAttribute: "fLaC", channels: "8") + for audio in [apac, flac] { + let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: audio, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.playbackTime = 600 + session.trackVideoTime(seconds: 641.4) + for start in [600.0, 602, 604] { + session.closeSegment(nextStartTime: start) + } + if audio.mediaPlaylistName != nil { + session.writeAudioInitSegment(Data([0])) + session.appendAudioSegment(Data([1]), duration: 3.2) + } + session.trackAudioTime(seconds: 603.4, duration: 1, streamEnd: 605.4) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 3, accuracy: 0.001) + // The last packet reaches the track's declared end, so a long video-only tail can pause. + session.trackAudioTime(seconds: 604.4, duration: 1, streamEnd: 605.4) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 40, accuracy: 0.001) + session.playbackTime = 620 + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 20, accuracy: 0.001) + session.finish(reachedEnd: false) + } + } + + func testAudioEndUsesTrackDurationOrMatroskaEndTag() throws { + let formatContext = avformat_alloc_context() + defer { avformat_free_context(formatContext) } + let stream = try XCTUnwrap(avformat_new_stream(formatContext, nil)) + stream.pointee.time_base = AVRational(num: 1, den: 1000) + stream.pointee.start_time = 1000 + stream.pointee.duration = 5000 + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 6) + stream.pointee.start_time = Int64.min + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 15) + XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "00:00:03.250000000", 0), 0) + // Matroska stores an absolute end timestamp, not a duration to add to start_time. + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 3.25) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "mov,mp4,m4a,3gp,3g2,mj2"), 15) + stream.pointee.duration = Int64.min + XCTAssertNil(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "mpegts")) + stream.pointee.duration = 5000 + XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "unknown", 0), 0) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 15) + XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "bad:00:00:03.250", 0), 0) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 15) + stream.pointee.duration = Int64.min + XCTAssertNil(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm")) + } + + func testRemuxMeasuresHowFarItReadsAheadOnceReady() { + let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: nil, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.trackVideoTime(seconds: 641.4) + // Playback counts from the start of the file; a transport stream's own times start later. + session.playbackTime = 600 + XCTAssertNil(session.remuxedAhead(sourceStart: 1.4)) + for start in [600.0, 602, 604] { + session.closeSegment(nextStartTime: start) + } + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? 0, 40, accuracy: 0.001) + } +} diff --git a/Tests/LumenTests/SourceSwitchTest.swift b/Tests/LumenTests/SourceSwitchTest.swift index 4131d6a..9f7cfc0 100644 --- a/Tests/LumenTests/SourceSwitchTest.swift +++ b/Tests/LumenTests/SourceSwitchTest.swift @@ -151,6 +151,20 @@ class SourceSwitchTest: XCTestCase { wait(for: [drained], timeout: 2) } + @MainActor + func testLayerCanBeReleasedBeforePlaybackStarts() throws { + KSOptions.firstPlayerType = ColdOnlyFakeEngine.self + let url = try XCTUnwrap(URL(string: "https://example.com/a.m3u8")) + let options = KSOptions() + options.registerRemoteControll = false + weak var releasedLayer: KSPlayerLayer? + do { + let layer = KSPlayerLayer(url: url, isAutoPlay: false, options: options) + releasedLayer = layer + } + XCTAssertNil(releasedLayer) + } + @MainActor func testDefaultSwitchSourceFallsBackToColdPath() throws { KSOptions.firstPlayerType = ColdOnlyFakeEngine.self diff --git a/docs/03-engine-meplayer-demux-e-pipeline.md b/docs/03-engine-meplayer-demux-e-pipeline.md index a419c92..5661306 100644 --- a/docs/03-engine-meplayer-demux-e-pipeline.md +++ b/docs/03-engine-meplayer-demux-e-pipeline.md @@ -116,6 +116,7 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa | `ProAVSegment` / `ProAVPlaylist` | `ProAVPlaylist.swift` | Modelo de segmento e geração dos textos `master.m3u8`/`media.m3u8` | | `ProAVInitBoundaryScanner` | `ProAVInitBoundaryScanner.swift` | Box-walk ISOBMFF sobre o stream do muxer: acha a fronteira entre o init segment e o primeiro `moof` | | `ProAVAudioTranscoder` | `ProAVAudioTranscoder.swift` | Transcode de áudio não suportado pelo HLS para FLAC | +| `ProAVSpatialAudioTranscoder` | `ProAVSpatialAudioTranscoder.swift` | Conversão de TrueHD Atmos e DTS:X para APAC com o SpatialAudioBridge, servida como rendition de áudio separada | | `DOVIPacketRewriter` | `DOVIPacketRewriter.swift` | Reescrita de packet HEVC para converter Dolby Vision perfil 7 em 8.1 (libdovi) | ### Fluxo @@ -143,7 +144,9 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa Áudio, por `ProAVAudioStrategy`: EC-3, AC-3, AAC, FLAC e ALAC passam por **stream copy**; qualquer outro codec (DTS, TrueHD, PCM, Opus…) é **transcodificado para FLAC** por `ProAVAudioTranscoder` (S16 ou S32/24 bits, sample rate e layout de canais preservados, PTS regenerado por contagem de amostras). Se o transcoder não puder inicializar, a trilha de áudio é simplesmente descartada. -A trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. +No tvOS, iOS e macOS 26 ou mais novos, TrueHD com o perfil Atmos do FFmpeg e todo DTS-HD MA a 48 kHz tentam primeiro o **APAC** (o FFmpeg rotula algumas apresentações DTS:X como DTS-HD MA simples): `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Quando a conversão falha, a sessão falha com `spatialAudioFailure` e o `ProAVPlayer` reinicia o launch na posição atual pelo caminho FLAC. Só uma recusa de decodificação da biblioteca (trilha sem Atmos ou DTS:X, ou com metadata que ela não renderiza) fixa o FLAC para aquela trilha da fonte; outras falhas valem só para aquele launch, e o próximo seek tenta o APAC de novo. No tvOS e no iOS, enquanto o APAC toca no próprio aparelho (fora do AirPlay), um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. + +Nos demais casos, a trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. ### Conversão de Dolby Vision perfil 7 @@ -161,7 +164,7 @@ Perfil 7 é dual-layer (base layer + enhancement layer + RPU) e não tem sample - **Sem legendas embutidas**: o remux carrega apenas vídeo e uma trilha de áudio; legendas do container não entram na playlist. - **A sinalização de Atmos depende do primeiro packet de áudio**: o `dec3` só sai completo porque o `moov` é adiado (`+delay_moov`) até o muxer ter parseado um frame E-AC-3. Se a trilha mapeada não entregar um packet com sync word dentro da janela do primeiro segmento, a sessão falha de propósito — antes ela emitiria um `dec3` zerado. O `CHANNELS="16/JOC"` da playlist, por outro lado, vem do `codecpar.profile` (decoder), não do muxer: as duas metades da sinalização têm fontes independentes e podem, em tese, discordar. - **RPU só é manipulada no perfil 7**: em DV perfil 5 e 8 as RPUs seguem embutidas no bitstream HEVC, sem extração nem reinjeção, e quem as interpreta é o decoder da Apple. A única reescrita é a conversão de perfil 7 para 8.1 descrita acima. -- **Consumo de disco**: o workspace cresce enquanto o remux avança, e só é limpo na próxima inicialização do player. `switchSource` marca o diretório do launch descartado para remoção (`requestCleanup`), que acontece quando aquele remux termina de fechar. +- **Consumo de disco**: o workspace cresce enquanto o remux avança, e só é limpo na próxima inicialização do player. O remux lê no máximo `maxBufferDuration` à frente da reprodução, como o engine de decodificação: pausa acima disso e retoma abaixo da metade, com a posição vinda de um observer periódico do `AVPlayer`. `switchSource` marca o diretório do launch descartado para remoção (`requestCleanup`), que acontece quando aquele remux termina de fechar. ## Cache de disco por byte-range