From d18069c7d5d156b0a7999b59f631f4872ddc17ca Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:34:30 +0200 Subject: [PATCH 01/36] feat: play truehd atmos and dts:x as apac through the proav remux The remux re-encoded TrueHD and DTS to FLAC, which keeps the channel bed but drops the Atmos and DTS:X objects. On tvOS, iOS and macOS 26, 48 kHz TrueHD and DTS-HD MA tracks now go to SpatialAudioBridge instead, and its 7.1.4 APAC fragments are served as a separate audio rendition on the muxer's timeline. A track without Atmos or DTS:X, or with metadata the library cannot render, fails the launch as decoding starts; the player restarts it at the same position on the FLAC path. AtmosOutputActivator keeps the receiver in Atmos while APAC plays. --- Package.swift | 4 + Sources/Lumen/MEPlayer/MEPlayerItem.swift | 26 ++- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 57 ++++- Sources/Lumen/MEPlayer/ProAVPlaylist.swift | 5 + .../Lumen/MEPlayer/ProAVRemuxSession.swift | 70 +++++++ .../ProAVSpatialAudioTranscoder.swift | 194 ++++++++++++++++++ 6 files changed, 352 insertions(+), 4 deletions(-) create mode 100644 Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift diff --git a/Package.swift b/Package.swift index 9ed4027..670aa6c 100644 --- a/Package.swift +++ b/Package.swift @@ -21,6 +21,9 @@ let package = Package( name: "Lumen", dependencies: [ .product(name: "FFmpegKit", package: "FFmpegKit"), + // The core ships arm64 slices for Apple TV, iPhone, iPad and Mac, without Mac Catalyst. + .product(name: "SpatialAudioBridge", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), + .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), // .product(name: "Libass", package: "FFmpegKit"), // .product(name: "Libmpv", package: "FFmpegKit"), "DisplayCriteria", @@ -43,4 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.2.0"), ] diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 75cb2d1..54b0056 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -25,6 +25,7 @@ public final class MEPlayerItem: Sendable { private let remuxSession: ProAVRemuxSession? private var remuxTranscoder: ProAVAudioTranscoder? private var remuxTranscodeStreamIndex: Int? + private var remuxSpatialAudio: ProAVSpatialAudioTranscoder? private var remuxCompleted = false private var remuxHeaderWritten = false private var remuxMoovWritten = false @@ -357,7 +358,10 @@ extension MEPlayerItem { let audioAssetTrack = audios.first { preferredAudioTrackID == nil ? $0.isEnabled : $0.trackID == preferredAudioTrackID } ?? audios.first { $0.isEnabled } ?? audios.first if let audioAssetTrack, let inputAudioStream = formatCtx.pointee.streams[Int(audioAssetTrack.trackID)] { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) - if strategy.copiesBitstream { + if session.allowsSpatialAudio, let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { + remuxSpatialAudio = spatialAudio + audioSignaling = spatialAudio.signaling + } else if strategy.copiesBitstream { if let outputAudioStream = avformat_new_stream(outputFormatCtx, nil) { avcodec_parameters_copy(outputAudioStream.pointee.codecpar, inputAudioStream.pointee.codecpar) outputAudioStream.pointee.codecpar.pointee.codec_tag = 0 @@ -381,7 +385,7 @@ extension MEPlayerItem { } else { KSLog("ProAV audio track skipped: flac transcode unavailable") } - if streamMapping[Int(audioAssetTrack.trackID)] != nil { + if streamMapping[Int(audioAssetTrack.trackID)] != nil || remuxSpatialAudio != nil { audios.forEach { $0.isEnabled = $0.trackID == audioAssetTrack.trackID } } } @@ -437,6 +441,10 @@ extension MEPlayerItem { private func writeProAVPacket(corePacket: UnsafeMutablePointer, outputFormatCtx: UnsafeMutablePointer, formatCtx: UnsafeMutablePointer, session: ProAVRemuxSession) { guard !session.isFailed else { return } let index = Int(corePacket.pointee.stream_index) + if let remuxSpatialAudio, remuxSpatialAudio.streamIndex == index { + remuxSpatialAudio.append(packet: corePacket) + return + } guard let outputIndex = streamMapping[index], let inputStream = formatCtx.pointee.streams[index], let outputStream = outputFormatCtx.pointee.streams[outputIndex] @@ -496,6 +504,14 @@ extension MEPlayerItem { outputPacket.pointee.stream_index = Int32(outputIndex) av_packet_rescale_ts(outputPacket, inputStream.pointee.time_base, outputStream.pointee.time_base) outputPacket.pointee.pos = -1 + if let remuxSpatialAudio, !remuxSpatialAudio.hasTimeline, outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_VIDEO, outputPacket.pointee.pts != Int64.min { + // The muxer shifts its timeline so that the first packet decodes at zero, deriving a missing + // decode time from the reorder delay as it does. The APAC rendition follows the same shift. + let timebase = Timebase(outputStream.pointee.time_base) + let pts = outputPacket.pointee.pts + let dts = outputPacket.pointee.dts == Int64.min ? pts - Int64(outputStream.pointee.codecpar.pointee.video_delay) * outputPacket.pointee.duration : outputPacket.pointee.dts + remuxSpatialAudio.begin(origin: timebase.cmtime(for: dts), videoStart: timebase.cmtime(for: pts)) + } let ret = av_write_frame(outputFormatCtx, outputPacket) if ret < 0 { KSLog("ProAV remux can not av_write_frame") @@ -540,6 +556,12 @@ extension MEPlayerItem { private func finishRemux(reachedEnd: Bool) { guard let remuxSession, !remuxCompleted else { return } remuxCompleted = true + if reachedEnd, !remuxSession.isFailed { + remuxSpatialAudio?.finish() + } else { + remuxSpatialAudio?.cancel() + } + remuxSpatialAudio = nil if let outputFormatCtx, remuxHeaderWritten { if reachedEnd, let remuxTranscoder, let streamIndex = remuxTranscodeStreamIndex, let outputIndex = streamMapping[streamIndex], diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 9240196..95322f9 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -3,6 +3,9 @@ import Foundation #if os(tvOS) || os(xrOS) import DisplayCriteria #endif +#if canImport(SpatialAudioPlayback) +import SpatialAudioPlayback +#endif #if canImport(UIKit) import UIKit #else @@ -30,6 +33,8 @@ public final class ProAVPlayer { private var delegateProxy: ProAVLaunchDelegateProxy? private var pendingSourceSwitch: PendingSourceSwitch? private var preferredAudioTrackID: Int32? + private var allowsSpatialAudio = true + private var atmosOutput: AnyObject? private var reportedReady = false private var didFail = false public weak var delegate: MediaPlayerDelegate? @@ -80,6 +85,7 @@ public final class ProAVPlayer { let configuration = ProAVRemuxSession.Configuration(directory: directory, targetSegmentDuration: ProAVPlayer.segmentDuration, minimumSegmentsBeforeReady: ProAVPlayer.minimumSegmentsBeforeReady) let session = ProAVRemuxSession(configuration: configuration) session.preferredAudioTrackID = audioTrackID + session.allowsSpatialAudio = allowsSpatialAudio options.startPlayTime = time let item = MEPlayerItem(url: url, options: options, remuxSession: session) let proxy = ProAVLaunchDelegateProxy(target: self, launch: launchIndex) @@ -98,7 +104,7 @@ public final class ProAVPlayer { launch.session.onFailure = { [weak self, weak session = launch.session] sessionError in runOnMainThread { guard let self, let session, self.session === session else { return } - self.fail(error: sessionError) + self.sessionDidFail(session, error: sessionError) } } session = launch.session @@ -133,10 +139,38 @@ public final class ProAVPlayer { private func attach(baseURL: URL, directoryName: String) { guard !didFail else { return } let masterURL = baseURL.appendingPathComponent(directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) + updateAtmosOutput() innerPlayer.replace(url: masterURL, options: options) innerPlayer.prepareToPlay() } + /// Keeps the receiver in Dolby Atmos while the active launch plays APAC, which alone does not switch it. + private func updateAtmosOutput() { + #if canImport(SpatialAudioPlayback) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { return } + guard session?.playsSpatialAudio == true else { + detachAtmosOutput() + return + } + guard atmosOutput == nil else { return } + #if !os(macOS) + try? AVAudioSession.sharedInstance().setSupportsMultichannelContent(true) + #endif + let activator = AtmosOutputActivator() + activator.attach(to: innerPlayer.player) + atmosOutput = activator + #endif + } + + private func detachAtmosOutput() { + #if canImport(SpatialAudioPlayback) + if #available(macOS 26, iOS 26, tvOS 26, *) { + (atmosOutput as? AtmosOutputActivator)?.detach() + } + #endif + atmosOutput = nil + } + private func applyDisplayCriteria() { #if os(tvOS) || os(xrOS) guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager, @@ -151,6 +185,20 @@ public final class ProAVPlayer { #endif } + private func sessionDidFail(_ session: ProAVRemuxSession, error: NSError) { + guard session.spatialAudioFailed else { + fail(error: error) + return + } + // The track has no Atmos or DTS:X, or metadata the library cannot render, so it plays through + // the FLAC transcode from here on. + KSLog(level: .info, "\(error.localizedDescription); falling back to FLAC") + allowsSpatialAudio = false + let seekCompletion = pendingSeekCompletion + pendingSeekCompletion = nil + restart(at: currentPlaybackTime, completion: seekCompletion) + } + private func fail(error: NSError) { abortPendingSourceSwitch() guard !didFail else { return } @@ -159,6 +207,7 @@ public final class ProAVPlayer { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil + detachAtmosOutput() innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -220,7 +269,7 @@ public final class ProAVPlayer { pending.launch.session.onFailure = { [weak self, weak session = pending.launch.session] sessionError in runOnMainThread { guard let self, let session, self.session === session else { return } - self.fail(error: sessionError) + self.sessionDidFail(session, error: sessionError) } } previousItem?.shutdown() @@ -228,6 +277,7 @@ public final class ProAVPlayer { if session?.videoSignaling?.preferredDynamicRange != previousDynamicRange { applyDisplayCriteria() } + updateAtmosOutput() pending.completion(true) } @@ -317,6 +367,7 @@ extension ProAVPlayer: MediaPlayerProtocol { didFail = false reportedReady = false preferredAudioTrackID = nil + allowsSpatialAudio = true self.url = url self.options = options } @@ -363,6 +414,7 @@ extension ProAVPlayer: MediaPlayerProtocol { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil + detachAtmosOutput() innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -399,6 +451,7 @@ extension ProAVPlayer: MediaPlayerProtocol { guard track.mediaType == .audio else { return } guard preferredAudioTrackID != track.trackID else { return } preferredAudioTrackID = track.trackID + allowsSpatialAudio = true remuxItem?.assetTracks.filter { $0.mediaType == .audio }.forEach { $0.isEnabled = $0.trackID == track.trackID } restart(at: currentPlaybackTime, completion: nil) } diff --git a/Sources/Lumen/MEPlayer/ProAVPlaylist.swift b/Sources/Lumen/MEPlayer/ProAVPlaylist.swift index 6925798..394651a 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlaylist.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlaylist.swift @@ -127,6 +127,8 @@ struct ProAVVideoSignaling { struct ProAVAudioSignaling: Equatable { let codecsAttribute: String let channels: String? + /// The rendition's own media playlist; without one the audio is muxed into the video segments. + var mediaPlaylistName: String? = nil } enum ProAVAudioStrategy { @@ -203,6 +205,9 @@ enum ProAVPlaylist { if let channels = audio.channels { mediaAttributes.append("CHANNELS=\"\(channels)\"") } + if let mediaPlaylistName = audio.mediaPlaylistName { + mediaAttributes.append("URI=\"\(mediaPlaylistName)\"") + } lines.append("#EXT-X-MEDIA:" + mediaAttributes.joined(separator: ",")) } lines.append("#EXT-X-STREAM-INF:" + attributes.joined(separator: ",")) diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 4295250..b7c88ec 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -17,11 +17,15 @@ public final class ProAVRemuxSession: @unchecked Sendable { static let masterPlaylistName = "master.m3u8" static let mediaPlaylistName = "media.m3u8" static let initSegmentName = "init.mp4" + static let audioPlaylistName = "audio.m3u8" + static let audioInitSegmentName = "audio-init.mp4" let configuration: Configuration private let lock = NSLock() private var pendingEvents = [() -> Void]() private var _preferredAudioTrackID: Int32? + private var _allowsSpatialAudio = true + private var _spatialAudioFailed = false private var _onReady: ((URL) -> Void)? private var _onFailure: ((NSError) -> Void)? private var _videoSignaling: ProAVVideoSignaling? @@ -33,6 +37,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { private var initScanner = ProAVInitBoundaryScanner() private var initBoundaryFound = false private var segments = [ProAVSegment]() + private var audioSegments = [ProAVSegment]() private var segmentIndex = 0 private var segmentStart: Double? private var lastVideoSeconds: Double? @@ -64,6 +69,26 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + /// Whether TrueHD and DTS-HD MA tracks try the APAC conversion before FLAC. + var allowsSpatialAudio: Bool { + get { + withLock { _allowsSpatialAudio } + } + set { + withLock { _allowsSpatialAudio = newValue } + } + } + + /// Whether the audio plays as the separate APAC rendition. + var playsSpatialAudio: Bool { + withLock { audioSignaling?.mediaPlaylistName != nil } + } + + /// Whether the session failed because the APAC conversion did; the FLAC path can still play the track. + var spatialAudioFailed: Bool { + withLock { _spatialAudioFailed } + } + var onReady: ((URL) -> Void)? { get { withLock { _onReady } @@ -183,6 +208,9 @@ public final class ProAVRemuxSession: @unchecked Sendable { failLocked(NSError(description: "ProAV remux produced no segments")) } else { writeMediaPlaylistLocked(ended: true) + if audioSignaling?.mediaPlaylistName != nil { + writeAudioPlaylistLocked(ended: true) + } fireReadyIfNeededLocked(force: true) } } @@ -198,6 +226,33 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { failLocked(error) } } + func failSpatialAudio(_ error: NSError) { + withLock { + guard !failed else { return } + _spatialAudioFailed = true + failLocked(error) + } + } + + func writeAudioInitSegment(_ data: Data) { + withLock { + guard !finished, !failed else { return } + writeLocked(data: data, named: Self.audioInitSegmentName) + } + } + + func appendAudioSegment(_ data: Data, duration: Double) { + withLock { + guard !finished, !failed else { return } + let fileName = "audio\(audioSegments.count).m4s" + writeLocked(data: data, named: fileName) + guard !failed else { return } + audioSegments.append(ProAVSegment(fileName: fileName, duration: duration)) + writeAudioPlaylistLocked(ended: false) + fireReadyIfNeededLocked(force: false) + } + } + func requestCleanup() { withLock { cleanupWhenFinished = true @@ -257,9 +312,16 @@ public final class ProAVRemuxSession: @unchecked Sendable { writeLocked(text: media, to: mediaURL) } + private func writeAudioPlaylistLocked(ended: Bool) { + let media = ProAVPlaylist.media(targetDuration: configuration.targetSegmentDuration, initSegmentName: Self.audioInitSegmentName, segments: audioSegments, ended: ended) + writeLocked(text: media, to: configuration.directory.appendingPathComponent(Self.audioPlaylistName)) + } + private func fireReadyIfNeededLocked(force: Bool) { guard !readyFired, !failed, !segments.isEmpty else { return } guard force || segments.count >= configuration.minimumSegmentsBeforeReady else { return } + // A separate audio rendition needs a fragment too before the player can start. + guard audioSignaling?.mediaPlaylistName == nil || !audioSegments.isEmpty else { return } readyFired = true if let handler = _onReady { let url = masterURL @@ -283,6 +345,14 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + private func writeLocked(data: Data, named name: String) { + do { + try data.write(to: configuration.directory.appendingPathComponent(name)) + } catch { + failLocked(NSError(description: "ProAV segment write failed: \(name)")) + } + } + private func write(buffer: UnsafePointer?, size: Int32) -> Int32 { withLock { guard let buffer, size > 0 else { return size } diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift new file mode 100644 index 0000000..aea42d6 --- /dev/null +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -0,0 +1,194 @@ +import CoreMedia +import Foundation +import Libavcodec +import Libavformat +#if canImport(SpatialAudioBridge) +import SpatialAudioBridge +#endif + +/// Converts a TrueHD Atmos or DTS:X track to APAC, which the remux serves as a separate audio +/// rendition instead of the FLAC transcode. A track without Atmos or DTS:X fails the session as +/// decoding starts, and the player then plays it through the FLAC transcode. +/// +/// The read thread hands packets over and blocks while the conversion is behind, so the remux runs at +/// its pace. Fragments are placed on the muxer's timeline once `begin(origin:videoStart:)` says where +/// the video landed; packets that arrive before then are held. +final class ProAVSpatialAudioTranscoder: @unchecked Sendable { + private enum Input { + case timeline(origin: CMTime, videoStart: CMTime) + case packet(Data, time: CMTime?) + } + + let streamIndex: Int + let signaling: ProAVAudioSignaling + private let timebase: Timebase + private let queue: AsyncStream.Continuation + private let capacity = DispatchSemaphore(value: 64) + private let completion = DispatchSemaphore(value: 0) + private let lock = NSLock() + private var cancelled = false + private(set) var hasTimeline = false + + /// Returns nil below tvOS, iOS and macOS 26 and for any track other than 48 kHz TrueHD or DTS-HD + /// Master Audio, which keep the FLAC path. + init?(stream: UnsafeMutablePointer, session: ProAVRemuxSession) { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *), + let codecpar = stream.pointee.codecpar, + let codec = Self.codec(codecpar: codecpar.pointee) + else { + return nil + } + // The offline profile would hold the first fragment back for about 20 s of audio. + let configuration = APACConfiguration(liveEncoding: true) + guard let priming = try? configuration.encoderPrimingDuration() else { + return nil + } + streamIndex = Int(stream.pointee.index) + timebase = Timebase(stream.pointee.time_base) + signaling = ProAVAudioSignaling(codecsAttribute: configuration.layout.apacCodecString, channels: "\(configuration.layout.channelCount)", mediaPlaylistName: ProAVRemuxSession.audioPlaylistName) + let (inputs, queue) = AsyncStream.makeStream(of: Input.self) + self.queue = queue + Task.detached { [self] in + await convert(inputs, codec: codec, configuration: configuration, priming: priming, session: session) + } + #else + return nil + #endif + } + + private var isCancelled: Bool { + lock.lock() + defer { lock.unlock() } + return cancelled + } + + /// Places the audio on the muxer's timeline, which starts at `origin`, the first video decode + /// time, and starts it with the first video frame at `videoStart`. + func begin(origin: CMTime, videoStart: CMTime) { + hasTimeline = true + capacity.wait() + queue.yield(.timeline(origin: origin, videoStart: videoStart)) + } + + func append(packet: UnsafeMutablePointer) { + guard let data = packet.pointee.data, packet.pointee.size > 0 else { return } + let time = packet.pointee.pts == Int64.min ? nil : timebase.cmtime(for: packet.pointee.pts) + capacity.wait() + queue.yield(.packet(Data(bytes: data, count: Int(packet.pointee.size)), time: time)) + } + + /// Converts the rest and waits until the last fragment reached the session. + func finish() { + queue.finish() + completion.wait() + } + + func cancel() { + lock.lock() + cancelled = true + lock.unlock() + queue.finish() + } + + #if canImport(SpatialAudioBridge) + @available(macOS 26, iOS 26, tvOS 26, *) + static func codec(codecpar: AVCodecParameters) -> SpatialAudioCodec? { + guard codecpar.sample_rate == 48000 else { + return nil + } + // FFmpeg reports some DTS:X presentations as plain DTS-HD MA, so every lossless track is + // offered and the library tells them apart once it decodes. + switch codecpar.codec_id { + case AV_CODEC_ID_TRUEHD: + return .trueHDAtmos + case AV_CODEC_ID_DTS where [AV_PROFILE_DTS_HD_MA, AV_PROFILE_DTS_HD_MA_X, AV_PROFILE_DTS_HD_MA_X_IMAX].contains(codecpar.profile): + return .dtsX + default: + return nil + } + } + + @available(macOS 26, iOS 26, tvOS 26, *) + private func convert(_ inputs: AsyncStream, codec: SpatialAudioCodec, configuration: APACConfiguration, priming: CMTime, session: ProAVRemuxSession) async { + var transcoder: SpatialAudioTranscoder? + var timeline: (origin: CMTime, start: CMTime)? + var held = [(data: Data, time: CMTime?)]() + var failed = false + // Any library error ends the session at once, so the player restarts on the FLAC path. + func fail(_ error: Error) { + failed = true + guard !isCancelled else { return } + session.failSpatialAudio(NSError(description: "ProAV spatial audio failed: \(error)")) + } + func append(_ data: Data, time: CMTime?) async throws { + guard let timeline else { + held.append((data, time)) + return + } + if transcoder == nil { + // Decoding starts at a TrueHD major sync or a DTS core sync; earlier audio is discarded + // and a later start is padded with silence, so the audio begins with the video. + guard let time, Self.isSyncPoint(data, codec: codec) else { return } + var settings = configuration + settings.startTime = timeline.start + settings.inputStartTime = time - timeline.origin + transcoder = try SpatialAudioTranscoder(codec: codec, destination: .segments { segment in + switch segment.kind { + case .initialization: + session.writeAudioInitSegment(segment.data) + case .media: + session.appendAudioSegment(segment.data, duration: segment.duration?.seconds ?? 0) + } + }, configuration: settings) + } + try await transcoder?.append(data, time: time.map { $0 - timeline.origin }) + } + // A failed or cancelled conversion keeps draining, so the read thread never waits on it. + for await input in inputs { + defer { capacity.signal() } + guard !failed, !isCancelled else { continue } + do { + switch input { + case let .timeline(origin, videoStart): + // Fragments keep the encoder priming before the start, which cannot go below zero. + timeline = (origin, CMTimeMaximum(videoStart - origin, priming)) + let waiting = held + held = [] + for packet in waiting { + try await append(packet.data, time: packet.time) + } + case let .packet(data, time): + try await append(data, time: time) + } + } catch { + fail(error) + } + } + if isCancelled { + await transcoder?.cancel() + } else if !failed { + do { + guard let transcoder else { + throw SpatialAudioError.source("The track has no sync point to start from") + } + _ = try await transcoder.finish() + } catch { + fail(error) + } + } + completion.signal() + } + + @available(macOS 26, iOS 26, tvOS 26, *) + private static func isSyncPoint(_ data: Data, codec: SpatialAudioCodec) -> Bool { + let bytes = [UInt8](data.prefix(8)) + switch codec { + case .trueHDAtmos: + return bytes.count == 8 && bytes[4 ... 7] == [0xF8, 0x72, 0x6F, 0xBA] + case .dtsX: + return bytes.starts(with: [0x7F, 0xFE, 0x80, 0x01]) + } + } + #endif +} From bc3fec4ba9b4280a895fdfd27160e0897e7cda3c Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:34:31 +0200 Subject: [PATCH 02/36] test: cover the apac rendition, its readiness gate and the track gate --- Tests/LumenTests/ProAVSpatialAudioTest.swift | 80 ++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 Tests/LumenTests/ProAVSpatialAudioTest.swift diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift new file mode 100644 index 0000000..a5aa7f1 --- /dev/null +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -0,0 +1,80 @@ +import FFmpegKit +import Libavcodec +@testable import Lumen +#if canImport(SpatialAudioBridge) +import SpatialAudioBridge +#endif +import XCTest + +class ProAVSpatialAudioTest: XCTestCase { + private let apac = ProAVAudioSignaling(codecsAttribute: "apac.31.03", channels: "12", mediaPlaylistName: ProAVRemuxSession.audioPlaylistName) + + func testMasterPointsAtTheAPACRendition() { + let video = ProAVVideoSignaling(codecTag: "dvh1", codecsAttribute: "dvh1.08.06", videoRange: "PQ", supplementalCodecs: nil, preferredDynamicRange: .dolbyVision) + let master = ProAVPlaylist.master(mediaPlaylistName: "media.m3u8", video: video, audio: apac, bandwidth: 24_000_000, resolution: CGSize(width: 3840, height: 2160), frameRate: 23.976) + XCTAssertTrue(master.contains("#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"main\",NAME=\"Original\",DEFAULT=YES,AUTOSELECT=YES,CHANNELS=\"12\",URI=\"audio.m3u8\"\n")) + XCTAssertTrue(master.contains("CODECS=\"dvh1.08.06,apac.31.03\"")) + XCTAssertTrue(master.contains("AUDIO=\"main\"")) + } + + func testOnlyLosslessTrueHDAndDTSAreOffered() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + func codec(_ codecID: AVCodecID, profile: Int32, sampleRate: Int32 = 48000) -> SpatialAudioCodec? { + var codecpar = AVCodecParameters() + codecpar.codec_id = codecID + codecpar.profile = profile + codecpar.sample_rate = sampleRate + return ProAVSpatialAudioTranscoder.codec(codecpar: codecpar) + } + XCTAssertEqual(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_TRUEHD_ATMOS), .trueHDAtmos) + XCTAssertEqual(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_UNKNOWN), .trueHDAtmos) + for profile in [AV_PROFILE_DTS_HD_MA, AV_PROFILE_DTS_HD_MA_X, AV_PROFILE_DTS_HD_MA_X_IMAX] { + XCTAssertEqual(codec(AV_CODEC_ID_DTS, profile: profile), .dtsX) + } + for profile in [AV_PROFILE_DTS, AV_PROFILE_DTS_HD_HRA, AV_PROFILE_DTS_EXPRESS, AV_PROFILE_UNKNOWN] { + XCTAssertNil(codec(AV_CODEC_ID_DTS, profile: profile)) + } + XCTAssertNil(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_TRUEHD_ATMOS, sampleRate: 96000)) + XCTAssertNil(codec(AV_CODEC_ID_EAC3, profile: AV_PROFILE_EAC3_DDP_ATMOS)) + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + + func testReadyWaitsForTheAPACRendition() throws { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + let session = ProAVRemuxSession(configuration: .init(directory: directory)) + var readyCount = 0 + session.onReady = { _ in readyCount += 1 } + let video = ProAVVideoSignaling(codecTag: "avc1", codecsAttribute: "avc1.640028", videoRange: "SDR", supplementalCodecs: nil, preferredDynamicRange: .sdr) + XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + XCTAssertTrue(session.playsSpatialAudio) + for start in [0.0, 2, 4] { + session.closeSegment(nextStartTime: start) + } + XCTAssertEqual(readyCount, 0) + session.writeAudioInitSegment(Data([0])) + session.appendAudioSegment(Data([1]), duration: 3.2) + XCTAssertEqual(readyCount, 1) + session.finish(reachedEnd: true) + let playlist = try String(contentsOf: directory.appendingPathComponent(ProAVRemuxSession.audioPlaylistName), encoding: .utf8) + XCTAssertTrue(playlist.contains("#EXT-X-MAP:URI=\"audio-init.mp4\"\n#EXTINF:3.20000,\naudio0.m4s\n#EXT-X-ENDLIST\n")) + } + + func testOnlySpatialAudioFailuresFallBackToFLAC() { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + let failed = ProAVRemuxSession(configuration: .init(directory: directory)) + failed.fail(NSError(description: "segment write failed")) + failed.failSpatialAudio(NSError(description: "unsupported metadata")) + XCTAssertTrue(failed.isFailed) + XCTAssertFalse(failed.spatialAudioFailed) + let rejected = ProAVRemuxSession(configuration: .init(directory: directory)) + rejected.failSpatialAudio(NSError(description: "unsupported metadata")) + XCTAssertTrue(rejected.isFailed) + XCTAssertTrue(rejected.spatialAudioFailed) + } +} From 863f5830b26e0ef22b5db7648742e3c3cdf28ae0 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 13:34:31 +0200 Subject: [PATCH 03/36] docs: describe the apac path for truehd atmos and dts:x --- README.md | 3 ++- docs/03-engine-meplayer-demux-e-pipeline.md | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 51ab7f2..978d7ae 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- TrueHD and DTS are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone +- **TrueHD Atmos and DTS:X play as APAC** on tvOS, iOS and macOS 26 and later: [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen +- Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode - On tvOS, frame rate and dynamic range are handed to the display through `AVDisplayCriteria` — so the TV can switch mode to match the content, when the viewer has Match Content enabled diff --git a/docs/03-engine-meplayer-demux-e-pipeline.md b/docs/03-engine-meplayer-demux-e-pipeline.md index a419c92..81109c2 100644 --- a/docs/03-engine-meplayer-demux-e-pipeline.md +++ b/docs/03-engine-meplayer-demux-e-pipeline.md @@ -116,6 +116,7 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa | `ProAVSegment` / `ProAVPlaylist` | `ProAVPlaylist.swift` | Modelo de segmento e geração dos textos `master.m3u8`/`media.m3u8` | | `ProAVInitBoundaryScanner` | `ProAVInitBoundaryScanner.swift` | Box-walk ISOBMFF sobre o stream do muxer: acha a fronteira entre o init segment e o primeiro `moof` | | `ProAVAudioTranscoder` | `ProAVAudioTranscoder.swift` | Transcode de áudio não suportado pelo HLS para FLAC | +| `ProAVSpatialAudioTranscoder` | `ProAVSpatialAudioTranscoder.swift` | Conversão de TrueHD Atmos e DTS:X para APAC com o SpatialAudioBridge, servida como rendition de áudio separada | | `DOVIPacketRewriter` | `DOVIPacketRewriter.swift` | Reescrita de packet HEVC para converter Dolby Vision perfil 7 em 8.1 (libdovi) | ### Fluxo @@ -143,7 +144,9 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa Áudio, por `ProAVAudioStrategy`: EC-3, AC-3, AAC, FLAC e ALAC passam por **stream copy**; qualquer outro codec (DTS, TrueHD, PCM, Opus…) é **transcodificado para FLAC** por `ProAVAudioTranscoder` (S16 ou S32/24 bits, sample rate e layout de canais preservados, PTS regenerado por contagem de amostras). Se o transcoder não puder inicializar, a trilha de áudio é simplesmente descartada. -A trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. +No tvOS, iOS e macOS 26 ou mais novos, TrueHD e DTS-HD MA a 48 kHz tentam primeiro o **APAC**: `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Trilhas sem Atmos ou DTS:X, ou com metadata que a biblioteca não renderiza, falham a sessão assim que a decodificação começa (`spatialAudioFailed`); o `ProAVPlayer` então reinicia o launch na posição atual pelo caminho FLAC, que vale para o resto do item ou até outra trilha ser escolhida. Enquanto o APAC toca, um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. + +Nos demais casos, a trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. ### Conversão de Dolby Vision perfil 7 From 15567df6c34c00bf34b4efc8501d071531bc7d13 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:06:15 +0200 Subject: [PATCH 04/36] fix: carry the flac fallback through source switches and keep the play state A source-switch candidate whose track the library refused aborted the switch, although a cold launch of the same source plays it through the FLAC transcode; retry the candidate there instead. A refusal can also arrive long after playback started, because the remux runs ahead of it, so the fallback restart now resumes or stays paused as the viewer left it instead of following the seek autoplay option. --- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 72 +++++++++++++++++------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 95322f9..a114be9 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -34,6 +34,8 @@ public final class ProAVPlayer { private var pendingSourceSwitch: PendingSourceSwitch? private var preferredAudioTrackID: Int32? private var allowsSpatialAudio = true + /// Whether playback resumes after the APAC fallback restarts a launch that was already playing. + private var fallbackResumesPlayback: Bool? private var atmosOutput: AnyObject? private var reportedReady = false private var didFail = false @@ -52,6 +54,7 @@ public final class ProAVPlayer { let url: URL let options: KSOptions let startOffset: TimeInterval + let allowsSpatialAudio: Bool let completion: (Bool) -> Void var innerSwitchStarted = false } @@ -78,7 +81,7 @@ public final class ProAVPlayer { } } - private func makeLaunch(url: URL, options: KSOptions, at time: TimeInterval, audioTrackID: Int32?) -> RemuxLaunch { + private func makeLaunch(url: URL, options: KSOptions, at time: TimeInterval, audioTrackID: Int32?, allowsSpatialAudio: Bool) -> RemuxLaunch { launchIndex += 1 let directoryName = "launch\(launchIndex)" let directory = workspaceURL.appendingPathComponent(directoryName) @@ -94,7 +97,7 @@ public final class ProAVPlayer { } private func startRemux(at time: TimeInterval) { - let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: preferredAudioTrackID) + let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: preferredAudioTrackID, allowsSpatialAudio: allowsSpatialAudio) launch.session.onReady = { [weak self, weak session = launch.session, directoryName = launch.directoryName] _ in runOnMainThread { guard let self, let session, self.session === session else { return } @@ -196,7 +199,12 @@ public final class ProAVPlayer { allowsSpatialAudio = false let seekCompletion = pendingSeekCompletion pendingSeekCompletion = nil + let wasPlaying = isPlaying restart(at: currentPlaybackTime, completion: seekCompletion) + // The remux runs ahead of playback, so a refusal can arrive at any point; keep the viewer's choice. + if reportedReady, seekCompletion == nil { + fallbackResumesPlayback = wasPlaying + } } private func fail(error: NSError) { @@ -222,6 +230,7 @@ public final class ProAVPlayer { pendingSeekCompletion = nil previousCompletion?(false) pendingSeekCompletion = completion + fallbackResumesPlayback = nil didFail = false remuxItem?.delegate = nil innerPlayer.shutdown() @@ -230,6 +239,38 @@ public final class ProAVPlayer { startRemux(at: time) } + private func startSourceSwitch(url: URL, options: KSOptions, allowsSpatialAudio: Bool, completion: @escaping (Bool) -> Void) { + let time = currentPlaybackTime + let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: nil, allowsSpatialAudio: allowsSpatialAudio) + pendingSourceSwitch = PendingSourceSwitch(launch: launch, url: url, options: options, startOffset: time, allowsSpatialAudio: allowsSpatialAudio, completion: completion) + launch.session.onReady = { [weak self, weak session = launch.session] _ in + runOnMainThread { + guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } + self.pendingSourceSwitchDidBecomeReady() + } + } + launch.session.onFailure = { [weak self, weak session = launch.session] _ in + runOnMainThread { + guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } + self.pendingSourceSwitchDidFail(session) + } + } + launch.item.prepareToPlay() + } + + /// Retries a candidate whose track the library refused on the FLAC path, as a cold launch would. + private func pendingSourceSwitchDidFail(_ session: ProAVRemuxSession) { + guard let pending = pendingSourceSwitch, session.spatialAudioFailed, !pending.innerSwitchStarted else { + abortPendingSourceSwitch() + return + } + pendingSourceSwitch = nil + pending.launch.item.delegate = nil + pending.launch.item.shutdown() + pending.launch.session.requestCleanup() + startSourceSwitch(url: pending.url, options: pending.options, allowsSpatialAudio: false, completion: pending.completion) + } + private func pendingSourceSwitchDidBecomeReady() { guard let pending = pendingSourceSwitch else { return } guard let serverBaseURL else { @@ -260,6 +301,7 @@ public final class ProAVPlayer { url = pending.url options = pending.options preferredAudioTrackID = nil + allowsSpatialAudio = pending.allowsSpatialAudio session = pending.launch.session remuxItem = pending.launch.item delegateProxy = pending.launch.proxy @@ -386,22 +428,7 @@ extension ProAVPlayer: MediaPlayerProtocol { completion(false) return } - let time = currentPlaybackTime - let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: nil) - pendingSourceSwitch = PendingSourceSwitch(launch: launch, url: url, options: options, startOffset: time, completion: completion) - launch.session.onReady = { [weak self, weak session = launch.session] _ in - runOnMainThread { - guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } - self.pendingSourceSwitchDidBecomeReady() - } - } - launch.session.onFailure = { [weak self, weak session = launch.session] _ in - runOnMainThread { - guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } - self.abortPendingSourceSwitch() - } - } - launch.item.prepareToPlay() + startSourceSwitch(url: url, options: options, allowsSpatialAudio: allowsSpatialAudio, completion: completion) } public func cancelSourceSwitch() { @@ -463,7 +490,14 @@ extension ProAVPlayer: MediaPlayerDelegate { pendingSeekCompletion = nil if reportedReady { seekCompletion?(true) - if options.isSeekedAutoPlay { + if let resumesPlayback = fallbackResumesPlayback { + fallbackResumesPlayback = nil + if resumesPlayback { + play() + } else { + pause() + } + } else if options.isSeekedAutoPlay { play() } } else { From 4f0353a392c3cddc71bc692491203b184b4dd3ae Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:06:15 +0200 Subject: [PATCH 05/36] fix: time spatial audio and its video origin by dts when pts is missing The remux already falls back to the decode time for segment timing. Without the same fallback here, a stream that only carries decode times never started its APAC session and fell back to FLAC. --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 4 ++-- Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 54b0056..f2ca68f 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -504,11 +504,11 @@ extension MEPlayerItem { outputPacket.pointee.stream_index = Int32(outputIndex) av_packet_rescale_ts(outputPacket, inputStream.pointee.time_base, outputStream.pointee.time_base) outputPacket.pointee.pos = -1 - if let remuxSpatialAudio, !remuxSpatialAudio.hasTimeline, outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_VIDEO, outputPacket.pointee.pts != Int64.min { + let pts = outputPacket.pointee.pts == Int64.min ? outputPacket.pointee.dts : outputPacket.pointee.pts + if let remuxSpatialAudio, !remuxSpatialAudio.hasTimeline, outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_VIDEO, pts != Int64.min { // The muxer shifts its timeline so that the first packet decodes at zero, deriving a missing // decode time from the reorder delay as it does. The APAC rendition follows the same shift. let timebase = Timebase(outputStream.pointee.time_base) - let pts = outputPacket.pointee.pts let dts = outputPacket.pointee.dts == Int64.min ? pts - Int64(outputStream.pointee.codecpar.pointee.video_delay) * outputPacket.pointee.duration : outputPacket.pointee.dts remuxSpatialAudio.begin(origin: timebase.cmtime(for: dts), videoStart: timebase.cmtime(for: pts)) } diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift index aea42d6..d1944f6 100644 --- a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -73,7 +73,8 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { func append(packet: UnsafeMutablePointer) { guard let data = packet.pointee.data, packet.pointee.size > 0 else { return } - let time = packet.pointee.pts == Int64.min ? nil : timebase.cmtime(for: packet.pointee.pts) + let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts + let time = timestamp == Int64.min ? nil : timebase.cmtime(for: timestamp) capacity.wait() queue.yield(.packet(Data(bytes: data, count: Int(packet.pointee.size)), time: time)) } From 24bfdd1ef8c231d4f9d1377d3744dc8808f70eed Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:06:15 +0200 Subject: [PATCH 06/36] docs: note that the spatial audio core has arm64 slices only --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 978d7ae..8f1d7ea 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,7 @@ There are two floors, and they are not the same one: Scrub thumbnails are tvOS-only, at any version. - Swift 5.9+ / Xcode 15+ +- SpatialAudioBridge, which converts TrueHD Atmos and DTS:X, ships arm64 slices only: build macOS apps for Apple silicon (`ARCHS=arm64`). Mac Catalyst builds leave it out and keep the FLAC path ## Installation From fe03345ddea5b18c34e18afd934c60ef04528772 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:13:13 +0200 Subject: [PATCH 07/36] fix: fall back to flac when the apac rendition ends without a fragment A launch that reached the end of the file before the first APAC fragment never became ready and never failed, so playback hung. Fail it as an APAC failure, which restarts it on the FLAC path. --- Sources/Lumen/MEPlayer/ProAVRemuxSession.swift | 4 ++++ Tests/LumenTests/ProAVSpatialAudioTest.swift | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index b7c88ec..5617f4f 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -206,6 +206,10 @@ public final class ProAVRemuxSession: @unchecked Sendable { if reachedEnd, !failed { if segments.isEmpty { failLocked(NSError(description: "ProAV remux produced no segments")) + } else if audioSignaling?.mediaPlaylistName != nil, audioSegments.isEmpty { + // The session could never become ready; the FLAC transcode can still play the track. + _spatialAudioFailed = true + failLocked(NSError(description: "ProAV spatial audio produced no fragments")) } else { writeMediaPlaylistLocked(ended: true) if audioSignaling?.mediaPlaylistName != nil { diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index a5aa7f1..b00a2e7 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -65,6 +65,19 @@ class ProAVSpatialAudioTest: XCTestCase { XCTAssertTrue(playlist.contains("#EXT-X-MAP:URI=\"audio-init.mp4\"\n#EXTINF:3.20000,\naudio0.m4s\n#EXT-X-ENDLIST\n")) } + func testEndingWithoutAPACFragmentsFallsBackToFLAC() { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: directory) } + let session = ProAVRemuxSession(configuration: .init(directory: directory)) + let video = ProAVVideoSignaling(codecTag: "avc1", codecsAttribute: "avc1.640028", videoRange: "SDR", supplementalCodecs: nil, preferredDynamicRange: .sdr) + XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.closeSegment(nextStartTime: 0) + session.closeSegment(nextStartTime: 2) + session.finish(reachedEnd: true) + XCTAssertTrue(session.isFailed) + XCTAssertTrue(session.spatialAudioFailed) + } + func testOnlySpatialAudioFailuresFallBackToFLAC() { let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) let failed = ProAVRemuxSession(configuration: .init(directory: directory)) From 9362ab88f00e1c47513dd822f7d1bab331730b4b Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:28:03 +0200 Subject: [PATCH 08/36] fix: give the apac and video playlists one target duration Apple's HLS authoring specification requires the audio and video playlists of a variant to use the same target duration. Each playlist took the target from its own longest segment, so 3.2 s APAC fragments beside 2 s video segments announced 4 and 2. Write both together from the longest segment of either. --- .../Lumen/MEPlayer/ProAVRemuxSession.swift | 24 +++++++++---------- Tests/LumenTests/ProAVSpatialAudioTest.swift | 4 ++++ 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 5617f4f..d46ab2d 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -184,7 +184,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { segmentStart = nextStartTime currentHandle = openFileLocked(named: segmentFileName(index: segmentIndex)) currentSegmentBytes = 0 - writeMediaPlaylistLocked(ended: false) + writePlaylistsLocked(ended: false) fireReadyIfNeededLocked(force: false) } } @@ -211,10 +211,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { _spatialAudioFailed = true failLocked(NSError(description: "ProAV spatial audio produced no fragments")) } else { - writeMediaPlaylistLocked(ended: true) - if audioSignaling?.mediaPlaylistName != nil { - writeAudioPlaylistLocked(ended: true) - } + writePlaylistsLocked(ended: true) fireReadyIfNeededLocked(force: true) } } @@ -252,7 +249,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { writeLocked(data: data, named: fileName) guard !failed else { return } audioSegments.append(ProAVSegment(fileName: fileName, duration: duration)) - writeAudioPlaylistLocked(ended: false) + writePlaylistsLocked(ended: false) fireReadyIfNeededLocked(force: false) } } @@ -311,14 +308,15 @@ public final class ProAVRemuxSession: @unchecked Sendable { segmentIndex += 1 } - private func writeMediaPlaylistLocked(ended: Bool) { - let media = ProAVPlaylist.media(targetDuration: configuration.targetSegmentDuration, initSegmentName: Self.initSegmentName, segments: segments, ended: ended) + /// Writes the video playlist and the separate audio rendition's, which HLS requires to share one + /// target duration, so a longer segment in either raises both. + private func writePlaylistsLocked(ended: Bool) { + let targetDuration = max(configuration.targetSegmentDuration, (segments + audioSegments).map(\.duration).max() ?? 0) + let media = ProAVPlaylist.media(targetDuration: targetDuration, initSegmentName: Self.initSegmentName, segments: segments, ended: ended) writeLocked(text: media, to: mediaURL) - } - - private func writeAudioPlaylistLocked(ended: Bool) { - let media = ProAVPlaylist.media(targetDuration: configuration.targetSegmentDuration, initSegmentName: Self.audioInitSegmentName, segments: audioSegments, ended: ended) - writeLocked(text: media, to: configuration.directory.appendingPathComponent(Self.audioPlaylistName)) + guard audioSignaling?.mediaPlaylistName != nil else { return } + let audio = ProAVPlaylist.media(targetDuration: targetDuration, initSegmentName: Self.audioInitSegmentName, segments: audioSegments, ended: ended) + writeLocked(text: audio, to: configuration.directory.appendingPathComponent(Self.audioPlaylistName)) } private func fireReadyIfNeededLocked(force: Bool) { diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index b00a2e7..5806e7d 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -63,6 +63,10 @@ class ProAVSpatialAudioTest: XCTestCase { session.finish(reachedEnd: true) let playlist = try String(contentsOf: directory.appendingPathComponent(ProAVRemuxSession.audioPlaylistName), encoding: .utf8) XCTAssertTrue(playlist.contains("#EXT-X-MAP:URI=\"audio-init.mp4\"\n#EXTINF:3.20000,\naudio0.m4s\n#EXT-X-ENDLIST\n")) + // HLS requires the audio and video playlists of a variant to share one target duration. + let media = try String(contentsOf: directory.appendingPathComponent(ProAVRemuxSession.mediaPlaylistName), encoding: .utf8) + XCTAssertTrue(playlist.contains("#EXT-X-TARGETDURATION:4\n")) + XCTAssertTrue(media.contains("#EXT-X-TARGETDURATION:4\n")) } func testEndingWithoutAPACFragmentsFallsBackToFLAC() { From 21805513a789f143e6603f1590471a7d2e630417 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:28:04 +0200 Subject: [PATCH 09/36] fix: keep the play state only for a launch that played, and retry late switch refusals A refusal that arrived before its launch became ready, such as after a track change, sampled the stopped replacement and paused playback; such a launch now restarts as the one before it would. A source-switch candidate refused after the inner player started loading it now abandons that load and retries on FLAC instead of failing the switch. --- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index a114be9..ddae061 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -199,12 +199,11 @@ public final class ProAVPlayer { allowsSpatialAudio = false let seekCompletion = pendingSeekCompletion pendingSeekCompletion = nil - let wasPlaying = isPlaying + // The remux runs ahead of playback, so a refusal can arrive while this launch plays: keep the + // viewer's choice then. A launch that never became ready restarts as the one before it would. + let resumesPlayback = isReadyToPlay && seekCompletion == nil ? isPlaying : nil restart(at: currentPlaybackTime, completion: seekCompletion) - // The remux runs ahead of playback, so a refusal can arrive at any point; keep the viewer's choice. - if reportedReady, seekCompletion == nil { - fallbackResumesPlayback = wasPlaying - } + fallbackResumesPlayback = resumesPlayback } private func fail(error: NSError) { @@ -260,11 +259,14 @@ public final class ProAVPlayer { /// Retries a candidate whose track the library refused on the FLAC path, as a cold launch would. private func pendingSourceSwitchDidFail(_ session: ProAVRemuxSession) { - guard let pending = pendingSourceSwitch, session.spatialAudioFailed, !pending.innerSwitchStarted else { + guard let pending = pendingSourceSwitch, session.spatialAudioFailed else { abortPendingSourceSwitch() return } pendingSourceSwitch = nil + if pending.innerSwitchStarted { + innerPlayer.abandonPendingSourceSwitch() + } pending.launch.item.delegate = nil pending.launch.item.shutdown() pending.launch.session.requestCleanup() From a96e5ff9adae5639d1a8d796e28303f144687b61 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 16:28:04 +0200 Subject: [PATCH 10/36] docs: state that building needs xcode 26 SpatialAudioBridge's manifest needs Swift 6 tools and its API the 26 SDKs, so Xcode 15 no longer builds Lumen. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8f1d7ea..9d66568 100644 --- a/README.md +++ b/README.md @@ -81,8 +81,8 @@ There are two floors, and they are not the same one: Scrub thumbnails are tvOS-only, at any version. -- Swift 5.9+ / Xcode 15+ -- SpatialAudioBridge, which converts TrueHD Atmos and DTS:X, ships arm64 slices only: build macOS apps for Apple silicon (`ARCHS=arm64`). Mac Catalyst builds leave it out and keep the FLAC path +- Xcode 26 or later: SpatialAudioBridge, which converts TrueHD Atmos and DTS:X, needs Swift 6 tools and the tvOS, iOS and macOS 26 SDKs to build. Lumen itself still compiles in the Swift 5 language mode +- SpatialAudioBridge's core ships arm64 slices only: build macOS apps for Apple silicon (`ARCHS=arm64`). Mac Catalyst builds leave it out and keep the FLAC path ## Installation From 0fae797e19dad21b943aba0c3a2dc70a2bf81fb5 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 18:37:34 +0200 Subject: [PATCH 11/36] build: move to spatialaudiobridge 1.3.0 1.3.0 turns TrueHD dialogue normalisation off by default, so TrueHD Atmos plays at Apple TV's native Atmos level. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 670aa6c..667fdf3 100644 --- a/Package.swift +++ b/Package.swift @@ -46,5 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), - .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.2.0"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.3.0"), ] From d6003bd73b22cb4220e20a8eb9eee27a9a1bf9a0 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:04:39 +0200 Subject: [PATCH 12/36] fix: keep a switch candidate's failure handler through the commit A failure queued while the switch committed ran the pending handler after the pending state was cleared, which dropped it and left the committed candidate on a failed remux. One handler now serves the candidate before and after the commit. --- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index ddae061..34dd720 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -248,10 +248,16 @@ public final class ProAVPlayer { self.pendingSourceSwitchDidBecomeReady() } } - launch.session.onFailure = { [weak self, weak session = launch.session] _ in + // One handler serves the candidate while it is pending and once it is committed, so a failure + // that arrives during the commit still reaches the active session. + launch.session.onFailure = { [weak self, weak session = launch.session] sessionError in runOnMainThread { - guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } - self.pendingSourceSwitchDidFail(session) + guard let self, let session else { return } + if self.pendingSourceSwitch?.launch.session === session { + self.pendingSourceSwitchDidFail(session) + } else if self.session === session { + self.sessionDidFail(session, error: sessionError) + } } } launch.item.prepareToPlay() @@ -310,12 +316,6 @@ public final class ProAVPlayer { activeLaunchIndex = pending.launch.index startOffset = pending.startOffset pending.launch.session.onReady = nil - pending.launch.session.onFailure = { [weak self, weak session = pending.launch.session] sessionError in - runOnMainThread { - guard let self, let session, self.session === session else { return } - self.sessionDidFail(session, error: sessionError) - } - } previousItem?.shutdown() previousSession?.requestCleanup() if session?.videoSignaling?.preferredDynamicRange != previousDynamicRange { From 3dcc75028c9a7567adef0ba0b8e97146d88ec979 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:37:39 +0200 Subject: [PATCH 13/36] fix: try apac again on a new source and honour commands during the fallback A source switch carried the current source's FLAC fallback to a different file, so its eligible track never tried APAC; the candidate now tries again and still retries on FLAC when refused. A play or pause while the FLAC fallback prepared was reversed when it became ready; such a command now replaces the state the fallback restores. --- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 34dd720..efe2618 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -417,10 +417,17 @@ extension ProAVPlayer: MediaPlayerProtocol { } public func play() { + // A command while the FLAC fallback prepares replaces the state it restores. + if fallbackResumesPlayback != nil { + fallbackResumesPlayback = true + } innerPlayer.play() } public func pause() { + if fallbackResumesPlayback != nil { + fallbackResumesPlayback = false + } innerPlayer.pause() } @@ -430,7 +437,8 @@ extension ProAVPlayer: MediaPlayerProtocol { completion(false) return } - startSourceSwitch(url: url, options: options, allowsSpatialAudio: allowsSpatialAudio, completion: completion) + // A new source tries APAC again; a refusal retries the candidate on FLAC. + startSourceSwitch(url: url, options: options, allowsSpatialAudio: true, completion: completion) } public func cancelSourceSwitch() { From 3d43a173d95ba599448b4376372638b64247edf9 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:50:26 +0200 Subject: [PATCH 14/36] fix: bound the audio taken in before the apac conversion starts Packets held for the video timeline returned their backpressure permit, and packets skipped while waiting for a sync point were never counted, so a stream that never started the conversion could buffer or read the whole file before falling back. Past 8 MB of such audio the session now fails as APAC, which restarts it on FLAC. --- Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift index d1944f6..a146a8f 100644 --- a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -115,6 +115,7 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { var transcoder: SpatialAudioTranscoder? var timeline: (origin: CMTime, start: CMTime)? var held = [(data: Data, time: CMTime?)]() + var unstartedBytes = 0 var failed = false // Any library error ends the session at once, so the player restarts on the FLAC path. func fail(_ error: Error) { @@ -160,6 +161,14 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { try await append(packet.data, time: packet.time) } case let .packet(data, time): + // Audio held for the timeline or skipped to a sync point is bounded, so a stream that + // never starts the conversion falls back to FLAC instead of buffering the file. + if transcoder == nil { + unstartedBytes += data.count + guard unstartedBytes <= 8 << 20 else { + throw SpatialAudioError.source("No sync point or video timeline within 8 MB of audio") + } + } try await append(data, time: time) } } catch { From 8d0a6897fb98b1992fd8a3ead11642dba296a501 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:04:35 +0200 Subject: [PATCH 15/36] fix: tighten the apac fallback, gate and read-ahead after review - Only a decoding refusal from the library keeps a track on FLAC, per track of the source. Encoding, CoreMedia and start-up failures, and a rendition that ends without a fragment, fall back for that launch only, so the next seek tries APAC again. Hot source switches carry the same policy. - TrueHD converts only when FFmpeg labels it Atmos; every DTS-HD MA track is still offered, since FFmpeg labels some DTS:X as plain MA. - A late APAC failure no longer restarts a player that already failed. - The remux reads at most the buffer duration ahead of playback, as the decoding engine does, so starting or seeking no longer converts the rest of the film. ProAVPlayer feeds the playback position to the session. - An APAC fragment without a duration fails the conversion instead of writing EXTINF:0. - The muxer timeline for the first video packet is a static function, and the sync point check and refusal test are reachable from tests. - Releasing the Atmos activator detaches it, and multichannel content is set on Lumen's audio-session queue and reset when APAC stops. --- Sources/Lumen/AVPlayer/KSOptions.swift | 11 +++ Sources/Lumen/MEPlayer/MEPlayerItem.swift | 23 +++-- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 93 ++++++++++--------- Sources/Lumen/MEPlayer/ProAVPlaylist.swift | 1 - .../Lumen/MEPlayer/ProAVRemuxSession.swift | 61 +++++++++--- .../ProAVSpatialAudioTranscoder.swift | 38 ++++++-- 6 files changed, 155 insertions(+), 72 deletions(-) diff --git a/Sources/Lumen/AVPlayer/KSOptions.swift b/Sources/Lumen/AVPlayer/KSOptions.swift index 136ca43..9601bba 100644 --- a/Sources/Lumen/AVPlayer/KSOptions.swift +++ b/Sources/Lumen/AVPlayer/KSOptions.swift @@ -555,6 +555,17 @@ public extension KSOptions { #endif } + /// Multichannel content is on while the remux plays APAC, as the Atmos output was tested. + static func setSupportsMultichannelContent(_ supports: Bool) { + #if !os(macOS) + audioSessionQueue.async { + if #available(tvOS 15.0, iOS 15.0, *) { + try? AVAudioSession.sharedInstance().setSupportsMultichannelContent(supports) + } + } + #endif + } + #if !os(macOS) static func isSpatialAudioEnabled(channelCount _: AVAudioChannelCount) -> Bool { if #available(tvOS 15.0, iOS 15.0, *) { diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index f2ca68f..7bc15c3 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -358,7 +358,7 @@ extension MEPlayerItem { let audioAssetTrack = audios.first { preferredAudioTrackID == nil ? $0.isEnabled : $0.trackID == preferredAudioTrackID } ?? audios.first { $0.isEnabled } ?? audios.first if let audioAssetTrack, let inputAudioStream = formatCtx.pointee.streams[Int(audioAssetTrack.trackID)] { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) - if session.allowsSpatialAudio, let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { + if session.spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { remuxSpatialAudio = spatialAudio audioSignaling = spatialAudio.signaling } else if strategy.copiesBitstream { @@ -504,13 +504,10 @@ extension MEPlayerItem { outputPacket.pointee.stream_index = Int32(outputIndex) av_packet_rescale_ts(outputPacket, inputStream.pointee.time_base, outputStream.pointee.time_base) outputPacket.pointee.pos = -1 - let pts = outputPacket.pointee.pts == Int64.min ? outputPacket.pointee.dts : outputPacket.pointee.pts - if let remuxSpatialAudio, !remuxSpatialAudio.hasTimeline, outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_VIDEO, pts != Int64.min { - // The muxer shifts its timeline so that the first packet decodes at zero, deriving a missing - // decode time from the reorder delay as it does. The APAC rendition follows the same shift. - let timebase = Timebase(outputStream.pointee.time_base) - let dts = outputPacket.pointee.dts == Int64.min ? pts - Int64(outputStream.pointee.codecpar.pointee.video_delay) * outputPacket.pointee.duration : outputPacket.pointee.dts - remuxSpatialAudio.begin(origin: timebase.cmtime(for: dts), videoStart: timebase.cmtime(for: pts)) + if let remuxSpatialAudio, !remuxSpatialAudio.hasTimeline, outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_VIDEO, + let timeline = ProAVSpatialAudioTranscoder.timeline(pts: outputPacket.pointee.pts, dts: outputPacket.pointee.dts, duration: outputPacket.pointee.duration, videoDelay: outputStream.pointee.codecpar.pointee.video_delay, timebase: Timebase(outputStream.pointee.time_base)) + { + remuxSpatialAudio.begin(origin: timeline.origin, videoStart: timeline.videoStart) } let ret = av_write_frame(outputFormatCtx, outputPacket) if ret < 0 { @@ -1101,7 +1098,15 @@ extension MEPlayerItem: MediaPlayback { extension MEPlayerItem: CodecCapacityDelegate { func codecDidChangeCapacity() { - guard remuxSession == nil else { + if let remuxSession { + // The remux reads ahead of playback by the buffer duration, as the decoding path buffers. + if let ahead = remuxSession.remuxedAhead { + if ahead > options.maxBufferDuration { + pause() + } else if ahead < options.maxBufferDuration / 2 { + resume() + } + } return } let loadingState = options.playable(capacitys: videoAudioTracks, isFirst: isFirst, isSeek: isSeek) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index efe2618..d577901 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -33,10 +33,19 @@ public final class ProAVPlayer { private var delegateProxy: ProAVLaunchDelegateProxy? private var pendingSourceSwitch: PendingSourceSwitch? private var preferredAudioTrackID: Int32? - private var allowsSpatialAudio = true + /// Audio tracks of this source that the library refused; they keep the FLAC path. + private var refusedSpatialAudioTrackIDs = Set() /// Whether playback resumes after the APAC fallback restarts a launch that was already playing. private var fallbackResumesPlayback: Bool? - private var atmosOutput: AnyObject? + private var playbackTimeObserver: Any? + /// The AtmosOutputActivator while a launch plays APAC; releasing it detaches it. + private var atmosOutput: AnyObject? { + didSet { + if (atmosOutput == nil) != (oldValue == nil) { + KSOptions.setSupportsMultichannelContent(atmosOutput != nil) + } + } + } private var reportedReady = false private var didFail = false public weak var delegate: MediaPlayerDelegate? @@ -54,7 +63,7 @@ public final class ProAVPlayer { let url: URL let options: KSOptions let startOffset: TimeInterval - let allowsSpatialAudio: Bool + let spatialAudio: ProAVRemuxSession.SpatialAudioPolicy let completion: (Bool) -> Void var innerSwitchStarted = false } @@ -67,6 +76,13 @@ public final class ProAVPlayer { ProAVPlayer.purgeWorkspaces() workspaceURL = ProAVPlayer.workspaceRoot.appendingPathComponent(UUID().uuidString) innerPlayer.delegate = self + // The remux reads only so far ahead of this position. + playbackTimeObserver = innerPlayer.player.addPeriodicTimeObserver(forInterval: CMTime(value: 1, timescale: 1), queue: .main) { [weak self] _ in + MainActor.assumeIsolated { + guard let self else { return } + self.session?.playbackTime = self.currentPlaybackTime + } + } } private static var workspaceRoot: URL { @@ -81,14 +97,14 @@ public final class ProAVPlayer { } } - private func makeLaunch(url: URL, options: KSOptions, at time: TimeInterval, audioTrackID: Int32?, allowsSpatialAudio: Bool) -> RemuxLaunch { + private func makeLaunch(url: URL, options: KSOptions, at time: TimeInterval, audioTrackID: Int32?, spatialAudio: ProAVRemuxSession.SpatialAudioPolicy) -> RemuxLaunch { launchIndex += 1 let directoryName = "launch\(launchIndex)" let directory = workspaceURL.appendingPathComponent(directoryName) let configuration = ProAVRemuxSession.Configuration(directory: directory, targetSegmentDuration: ProAVPlayer.segmentDuration, minimumSegmentsBeforeReady: ProAVPlayer.minimumSegmentsBeforeReady) let session = ProAVRemuxSession(configuration: configuration) session.preferredAudioTrackID = audioTrackID - session.allowsSpatialAudio = allowsSpatialAudio + session.spatialAudioPolicy = spatialAudio options.startPlayTime = time let item = MEPlayerItem(url: url, options: options, remuxSession: session) let proxy = ProAVLaunchDelegateProxy(target: self, launch: launchIndex) @@ -96,8 +112,9 @@ public final class ProAVPlayer { return RemuxLaunch(index: launchIndex, directoryName: directoryName, session: session, item: item, proxy: proxy) } - private func startRemux(at time: TimeInterval) { - let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: preferredAudioTrackID, allowsSpatialAudio: allowsSpatialAudio) + private func startRemux(at time: TimeInterval, spatialAudioEnabled: Bool = true) { + let spatialAudio = ProAVRemuxSession.SpatialAudioPolicy(refusedTrackIDs: refusedSpatialAudioTrackIDs, isEnabled: spatialAudioEnabled) + let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: preferredAudioTrackID, spatialAudio: spatialAudio) launch.session.onReady = { [weak self, weak session = launch.session, directoryName = launch.directoryName] _ in runOnMainThread { guard let self, let session, self.session === session else { return } @@ -152,28 +169,16 @@ public final class ProAVPlayer { #if canImport(SpatialAudioPlayback) guard #available(macOS 26, iOS 26, tvOS 26, *) else { return } guard session?.playsSpatialAudio == true else { - detachAtmosOutput() + atmosOutput = nil return } guard atmosOutput == nil else { return } - #if !os(macOS) - try? AVAudioSession.sharedInstance().setSupportsMultichannelContent(true) - #endif let activator = AtmosOutputActivator() activator.attach(to: innerPlayer.player) atmosOutput = activator #endif } - private func detachAtmosOutput() { - #if canImport(SpatialAudioPlayback) - if #available(macOS 26, iOS 26, tvOS 26, *) { - (atmosOutput as? AtmosOutputActivator)?.detach() - } - #endif - atmosOutput = nil - } - private func applyDisplayCriteria() { #if os(tvOS) || os(xrOS) guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager, @@ -189,20 +194,20 @@ public final class ProAVPlayer { } private func sessionDidFail(_ session: ProAVRemuxSession, error: NSError) { - guard session.spatialAudioFailed else { + guard !didFail else { return } + guard let failure = session.spatialAudioFailure else { fail(error: error) return } - // The track has no Atmos or DTS:X, or metadata the library cannot render, so it plays through - // the FLAC transcode from here on. KSLog(level: .info, "\(error.localizedDescription); falling back to FLAC") - allowsSpatialAudio = false + if case let .refused(trackID) = failure { + refusedSpatialAudioTrackIDs.insert(trackID) + } let seekCompletion = pendingSeekCompletion pendingSeekCompletion = nil - // The remux runs ahead of playback, so a refusal can arrive while this launch plays: keep the - // viewer's choice then. A launch that never became ready restarts as the one before it would. + // The remux runs ahead of playback, so the failure can arrive while this launch plays. let resumesPlayback = isReadyToPlay && seekCompletion == nil ? isPlaying : nil - restart(at: currentPlaybackTime, completion: seekCompletion) + restart(at: currentPlaybackTime, completion: seekCompletion, spatialAudioEnabled: false) fallbackResumesPlayback = resumesPlayback } @@ -214,7 +219,7 @@ public final class ProAVPlayer { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - detachAtmosOutput() + atmosOutput = nil innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -223,7 +228,7 @@ public final class ProAVPlayer { delegate?.finish(player: self, error: error) } - private func restart(at time: TimeInterval, completion: ((Bool) -> Void)?) { + private func restart(at time: TimeInterval, completion: ((Bool) -> Void)?, spatialAudioEnabled: Bool = true) { abortPendingSourceSwitch() let previousCompletion = pendingSeekCompletion pendingSeekCompletion = nil @@ -235,13 +240,13 @@ public final class ProAVPlayer { innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() - startRemux(at: time) + startRemux(at: time, spatialAudioEnabled: spatialAudioEnabled) } - private func startSourceSwitch(url: URL, options: KSOptions, allowsSpatialAudio: Bool, completion: @escaping (Bool) -> Void) { + private func startSourceSwitch(url: URL, options: KSOptions, spatialAudio: ProAVRemuxSession.SpatialAudioPolicy, completion: @escaping (Bool) -> Void) { let time = currentPlaybackTime - let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: nil, allowsSpatialAudio: allowsSpatialAudio) - pendingSourceSwitch = PendingSourceSwitch(launch: launch, url: url, options: options, startOffset: time, allowsSpatialAudio: allowsSpatialAudio, completion: completion) + let launch = makeLaunch(url: url, options: options, at: time, audioTrackID: nil, spatialAudio: spatialAudio) + pendingSourceSwitch = PendingSourceSwitch(launch: launch, url: url, options: options, startOffset: time, spatialAudio: spatialAudio, completion: completion) launch.session.onReady = { [weak self, weak session = launch.session] _ in runOnMainThread { guard let self, let session, self.pendingSourceSwitch?.launch.session === session else { return } @@ -263,12 +268,17 @@ public final class ProAVPlayer { launch.item.prepareToPlay() } - /// Retries a candidate whose track the library refused on the FLAC path, as a cold launch would. + /// Retries a candidate whose APAC conversion failed on the FLAC path, as a cold launch would. private func pendingSourceSwitchDidFail(_ session: ProAVRemuxSession) { - guard let pending = pendingSourceSwitch, session.spatialAudioFailed else { + guard let pending = pendingSourceSwitch, let failure = session.spatialAudioFailure else { abortPendingSourceSwitch() return } + var spatialAudio = pending.spatialAudio + spatialAudio.isEnabled = false + if case let .refused(trackID) = failure { + spatialAudio.refusedTrackIDs.insert(trackID) + } pendingSourceSwitch = nil if pending.innerSwitchStarted { innerPlayer.abandonPendingSourceSwitch() @@ -276,7 +286,7 @@ public final class ProAVPlayer { pending.launch.item.delegate = nil pending.launch.item.shutdown() pending.launch.session.requestCleanup() - startSourceSwitch(url: pending.url, options: pending.options, allowsSpatialAudio: false, completion: pending.completion) + startSourceSwitch(url: pending.url, options: pending.options, spatialAudio: spatialAudio, completion: pending.completion) } private func pendingSourceSwitchDidBecomeReady() { @@ -309,7 +319,7 @@ public final class ProAVPlayer { url = pending.url options = pending.options preferredAudioTrackID = nil - allowsSpatialAudio = pending.allowsSpatialAudio + refusedSpatialAudioTrackIDs = pending.spatialAudio.refusedTrackIDs session = pending.launch.session remuxItem = pending.launch.item delegateProxy = pending.launch.proxy @@ -411,13 +421,12 @@ extension ProAVPlayer: MediaPlayerProtocol { didFail = false reportedReady = false preferredAudioTrackID = nil - allowsSpatialAudio = true + refusedSpatialAudioTrackIDs = [] self.url = url self.options = options } public func play() { - // A command while the FLAC fallback prepares replaces the state it restores. if fallbackResumesPlayback != nil { fallbackResumesPlayback = true } @@ -437,8 +446,7 @@ extension ProAVPlayer: MediaPlayerProtocol { completion(false) return } - // A new source tries APAC again; a refusal retries the candidate on FLAC. - startSourceSwitch(url: url, options: options, allowsSpatialAudio: true, completion: completion) + startSourceSwitch(url: url, options: options, spatialAudio: ProAVRemuxSession.SpatialAudioPolicy(), completion: completion) } public func cancelSourceSwitch() { @@ -451,7 +459,7 @@ extension ProAVPlayer: MediaPlayerProtocol { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - detachAtmosOutput() + atmosOutput = nil innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -488,7 +496,6 @@ extension ProAVPlayer: MediaPlayerProtocol { guard track.mediaType == .audio else { return } guard preferredAudioTrackID != track.trackID else { return } preferredAudioTrackID = track.trackID - allowsSpatialAudio = true remuxItem?.assetTracks.filter { $0.mediaType == .audio }.forEach { $0.isEnabled = $0.trackID == track.trackID } restart(at: currentPlaybackTime, completion: nil) } diff --git a/Sources/Lumen/MEPlayer/ProAVPlaylist.swift b/Sources/Lumen/MEPlayer/ProAVPlaylist.swift index 394651a..3f97581 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlaylist.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlaylist.swift @@ -127,7 +127,6 @@ struct ProAVVideoSignaling { struct ProAVAudioSignaling: Equatable { let codecsAttribute: String let channels: String? - /// The rendition's own media playlist; without one the audio is muxed into the video segments. var mediaPlaylistName: String? = nil } diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index d46ab2d..823e106 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -20,12 +20,32 @@ public final class ProAVRemuxSession: @unchecked Sendable { static let audioPlaylistName = "audio.m3u8" static let audioInitSegmentName = "audio-init.mp4" + /// Which audio tracks of a launch try the APAC conversion before FLAC. + struct SpatialAudioPolicy: Equatable { + /// Tracks the library refused, which keep the FLAC path for their source. + var refusedTrackIDs = Set() + /// Off for a launch that retries on FLAC after the conversion failed without refusing the track. + var isEnabled = true + + func allows(trackID: Int32) -> Bool { + isEnabled && !refusedTrackIDs.contains(trackID) + } + } + + enum SpatialAudioFailure: Equatable { + /// The library refused the track, as it does for TrueHD without Atmos or DTS-HD MA without DTS:X. + case refused(trackID: Int32) + /// The conversion failed for this launch only. + case failed + } + let configuration: Configuration private let lock = NSLock() private var pendingEvents = [() -> Void]() private var _preferredAudioTrackID: Int32? - private var _allowsSpatialAudio = true - private var _spatialAudioFailed = false + private var _spatialAudioPolicy = SpatialAudioPolicy() + private var _spatialAudioFailure: SpatialAudioFailure? + private var _playbackTime: TimeInterval? private var _onReady: ((URL) -> Void)? private var _onFailure: ((NSError) -> Void)? private var _videoSignaling: ProAVVideoSignaling? @@ -69,13 +89,12 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } - /// Whether TrueHD and DTS-HD MA tracks try the APAC conversion before FLAC. - var allowsSpatialAudio: Bool { + var spatialAudioPolicy: SpatialAudioPolicy { get { - withLock { _allowsSpatialAudio } + withLock { _spatialAudioPolicy } } set { - withLock { _allowsSpatialAudio = newValue } + withLock { _spatialAudioPolicy = newValue } } } @@ -84,9 +103,27 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { audioSignaling?.mediaPlaylistName != nil } } - /// Whether the session failed because the APAC conversion did; the FLAC path can still play the track. - var spatialAudioFailed: Bool { - withLock { _spatialAudioFailed } + /// Set when the session failed because the APAC conversion did; the FLAC path can still play the track. + var spatialAudioFailure: SpatialAudioFailure? { + withLock { _spatialAudioFailure } + } + + /// The player's position, in source time, which the remux reads ahead of. + var playbackTime: TimeInterval? { + get { + withLock { _playbackTime } + } + set { + withLock { _playbackTime = newValue } + } + } + + /// How far the remuxed video runs ahead of playback, once both are known. + var remuxedAhead: TimeInterval? { + withLock { + guard let lastVideoSeconds, let _playbackTime else { return nil } + return lastVideoSeconds - _playbackTime + } } var onReady: ((URL) -> Void)? { @@ -208,7 +245,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { failLocked(NSError(description: "ProAV remux produced no segments")) } else if audioSignaling?.mediaPlaylistName != nil, audioSegments.isEmpty { // The session could never become ready; the FLAC transcode can still play the track. - _spatialAudioFailed = true + _spatialAudioFailure = .failed failLocked(NSError(description: "ProAV spatial audio produced no fragments")) } else { writePlaylistsLocked(ended: true) @@ -227,10 +264,10 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { failLocked(error) } } - func failSpatialAudio(_ error: NSError) { + func failSpatialAudio(_ error: NSError, refusedTrackID: Int32?) { withLock { guard !failed else { return } - _spatialAudioFailed = true + _spatialAudioFailure = refusedTrackID.map { .refused(trackID: $0) } ?? .failed failLocked(error) } } diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift index a146a8f..34c7029 100644 --- a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -71,6 +71,17 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { queue.yield(.timeline(origin: origin, videoStart: videoStart)) } + /// The muxer's timeline for its first video packet: the muxer shifts it so that the packet decodes + /// at zero, deriving a missing decode time from the reorder delay as it does. Nil without any time. + static func timeline(pts: Int64, dts: Int64, duration: Int64, videoDelay: Int32, timebase: Timebase) -> (origin: CMTime, videoStart: CMTime)? { + let start = pts == Int64.min ? dts : pts + guard start != Int64.min else { + return nil + } + let origin = dts == Int64.min ? start - Int64(videoDelay) * duration : dts + return (timebase.cmtime(for: origin), timebase.cmtime(for: start)) + } + func append(packet: UnsafeMutablePointer) { guard let data = packet.pointee.data, packet.pointee.size > 0 else { return } let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts @@ -98,10 +109,10 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { guard codecpar.sample_rate == 48000 else { return nil } - // FFmpeg reports some DTS:X presentations as plain DTS-HD MA, so every lossless track is - // offered and the library tells them apart once it decodes. + // FFmpeg reports some DTS:X presentations as plain DTS-HD MA, so every one is offered and the + // library tells them apart once it decodes. TrueHD goes by FFmpeg's Atmos label. switch codecpar.codec_id { - case AV_CODEC_ID_TRUEHD: + case AV_CODEC_ID_TRUEHD where codecpar.profile == AV_PROFILE_TRUEHD_ATMOS: return .trueHDAtmos case AV_CODEC_ID_DTS where [AV_PROFILE_DTS_HD_MA, AV_PROFILE_DTS_HD_MA_X, AV_PROFILE_DTS_HD_MA_X_IMAX].contains(codecpar.profile): return .dtsX @@ -117,11 +128,12 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { var held = [(data: Data, time: CMTime?)]() var unstartedBytes = 0 var failed = false - // Any library error ends the session at once, so the player restarts on the FLAC path. + // Any error ends the session at once, so the player restarts on the FLAC path; a decoding + // refusal keeps the track there. func fail(_ error: Error) { failed = true guard !isCancelled else { return } - session.failSpatialAudio(NSError(description: "ProAV spatial audio failed: \(error)")) + session.failSpatialAudio(NSError(description: "ProAV spatial audio failed: \(error)"), refusedTrackID: Self.refusesTrack(error) ? Int32(streamIndex) : nil) } func append(_ data: Data, time: CMTime?) async throws { guard let timeline else { @@ -140,7 +152,10 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { case .initialization: session.writeAudioInitSegment(segment.data) case .media: - session.appendAudioSegment(segment.data, duration: segment.duration?.seconds ?? 0) + guard let duration = segment.duration?.seconds, duration > 0 else { + throw SpatialAudioError.encoding("APAC fragment without a duration") + } + session.appendAudioSegment(segment.data, duration: duration) } }, configuration: settings) } @@ -190,8 +205,17 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { completion.signal() } + /// Whether the library refused the track itself, rather than failing to convert it this time. + @available(macOS 26, iOS 26, tvOS 26, *) + static func refusesTrack(_ error: Error) -> Bool { + if case .decoding? = error as? SpatialAudioError { + return true + } + return false + } + @available(macOS 26, iOS 26, tvOS 26, *) - private static func isSyncPoint(_ data: Data, codec: SpatialAudioCodec) -> Bool { + static func isSyncPoint(_ data: Data, codec: SpatialAudioCodec) -> Bool { let bytes = [UInt8](data.prefix(8)) switch codec { case .trueHDAtmos: From 8fca747660bbeea4b384e024fad2e1c7c2c660da Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:04:35 +0200 Subject: [PATCH 16/36] test: cover the muxer timeline, sync points, refusals and the 8 MB bound The timeline cases include the missing decode time after a seek with B-frames and a decode time alone. The bound test feeds a real conversion 8 MB of TrueHD without a major sync and expects a fallback for that launch only. --- Tests/LumenTests/ProAVSpatialAudioTest.swift | 145 ++++++++++++++++--- 1 file changed, 125 insertions(+), 20 deletions(-) diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index 5806e7d..6cc9a91 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -1,5 +1,7 @@ +import CoreMedia import FFmpegKit import Libavcodec +import Libavformat @testable import Lumen #if canImport(SpatialAudioBridge) import SpatialAudioBridge @@ -8,6 +10,13 @@ import XCTest class ProAVSpatialAudioTest: XCTestCase { private let apac = ProAVAudioSignaling(codecsAttribute: "apac.31.03", channels: "12", mediaPlaylistName: ProAVRemuxSession.audioPlaylistName) + private let video = ProAVVideoSignaling(codecTag: "avc1", codecsAttribute: "avc1.640028", videoRange: "SDR", supplementalCodecs: nil, preferredDynamicRange: .sdr) + + private func makeSession() -> ProAVRemuxSession { + let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + return ProAVRemuxSession(configuration: .init(directory: directory)) + } func testMasterPointsAtTheAPACRendition() { let video = ProAVVideoSignaling(codecTag: "dvh1", codecsAttribute: "dvh1.08.06", videoRange: "PQ", supplementalCodecs: nil, preferredDynamicRange: .dolbyVision) @@ -17,7 +26,7 @@ class ProAVSpatialAudioTest: XCTestCase { XCTAssertTrue(master.contains("AUDIO=\"main\"")) } - func testOnlyLosslessTrueHDAndDTSAreOffered() throws { + func testOnlyAtmosTrueHDAndLosslessDTSAreOffered() throws { #if canImport(SpatialAudioBridge) guard #available(macOS 26, iOS 26, tvOS 26, *) else { throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") @@ -30,7 +39,7 @@ class ProAVSpatialAudioTest: XCTestCase { return ProAVSpatialAudioTranscoder.codec(codecpar: codecpar) } XCTAssertEqual(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_TRUEHD_ATMOS), .trueHDAtmos) - XCTAssertEqual(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_UNKNOWN), .trueHDAtmos) + XCTAssertNil(codec(AV_CODEC_ID_TRUEHD, profile: AV_PROFILE_UNKNOWN)) for profile in [AV_PROFILE_DTS_HD_MA, AV_PROFILE_DTS_HD_MA_X, AV_PROFILE_DTS_HD_MA_X_IMAX] { XCTAssertEqual(codec(AV_CODEC_ID_DTS, profile: profile), .dtsX) } @@ -44,13 +53,96 @@ class ProAVSpatialAudioTest: XCTestCase { #endif } + func testConversionStartsAtASyncPoint() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + let majorSync = Data([0x12, 0x34, 0x56, 0x78, 0xF8, 0x72, 0x6F, 0xBA, 0x00]) + let mlpMajorSync = Data([0x12, 0x34, 0x56, 0x78, 0xF8, 0x72, 0x6F, 0xBB, 0x00]) + XCTAssertTrue(ProAVSpatialAudioTranscoder.isSyncPoint(majorSync, codec: .trueHDAtmos)) + XCTAssertFalse(ProAVSpatialAudioTranscoder.isSyncPoint(mlpMajorSync, codec: .trueHDAtmos)) + XCTAssertFalse(ProAVSpatialAudioTranscoder.isSyncPoint(majorSync.prefix(6), codec: .trueHDAtmos)) + XCTAssertTrue(ProAVSpatialAudioTranscoder.isSyncPoint(Data([0x7F, 0xFE, 0x80, 0x01, 0xFC]), codec: .dtsX)) + XCTAssertFalse(ProAVSpatialAudioTranscoder.isSyncPoint(Data([0xFE, 0x7F, 0x01, 0x80, 0xFC]), codec: .dtsX)) + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + + func testOnlyDecodingErrorsRefuseTheTrack() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + XCTAssertTrue(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.decoding("This TrueHD stream has no Atmos presentation"))) + XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.encoding("APAC encoder stopped"))) + XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.source("No sync point or video timeline within 8 MB of audio"))) + XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.coreMedia(-12780))) + XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(CancellationError())) + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + + func testTimelineFollowsTheMuxerShift() { + let timebase = Timebase(num: 1, den: 1000) + let demuxed = ProAVSpatialAudioTranscoder.timeline(pts: 1080, dts: 1000, duration: 40, videoDelay: 2, timebase: timebase) + XCTAssertEqual(demuxed?.origin, CMTime(value: 1000, timescale: 1000)) + XCTAssertEqual(demuxed?.videoStart, CMTime(value: 1080, timescale: 1000)) + // The first packet after a seek has no decode time, so the muxer derives it from the B-frame delay. + let afterSeek = ProAVSpatialAudioTranscoder.timeline(pts: 1000, dts: Int64.min, duration: 40, videoDelay: 2, timebase: timebase) + XCTAssertEqual(afterSeek?.origin, CMTime(value: 920, timescale: 1000)) + XCTAssertEqual(afterSeek?.videoStart, CMTime(value: 1000, timescale: 1000)) + let withoutReordering = ProAVSpatialAudioTranscoder.timeline(pts: 1000, dts: Int64.min, duration: 40, videoDelay: 0, timebase: timebase) + XCTAssertEqual(withoutReordering?.origin, CMTime(value: 1000, timescale: 1000)) + let decodeTimeOnly = ProAVSpatialAudioTranscoder.timeline(pts: Int64.min, dts: 500, duration: 40, videoDelay: 0, timebase: timebase) + XCTAssertEqual(decodeTimeOnly?.origin, CMTime(value: 500, timescale: 1000)) + XCTAssertEqual(decodeTimeOnly?.videoStart, CMTime(value: 500, timescale: 1000)) + XCTAssertNil(ProAVSpatialAudioTranscoder.timeline(pts: Int64.min, dts: Int64.min, duration: 40, videoDelay: 2, timebase: timebase)) + } + + func testConversionThatNeverStartsFallsBackForTheLaunch() throws { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *) else { + throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + } + let formatContext = avformat_alloc_context() + defer { avformat_free_context(formatContext) } + let stream = try XCTUnwrap(avformat_new_stream(formatContext, nil)) + stream.pointee.codecpar.pointee.codec_type = AVMEDIA_TYPE_AUDIO + stream.pointee.codecpar.pointee.codec_id = AV_CODEC_ID_TRUEHD + stream.pointee.codecpar.pointee.profile = AV_PROFILE_TRUEHD_ATMOS + stream.pointee.codecpar.pointee.sample_rate = 48000 + stream.pointee.time_base = AVRational(num: 1, den: 1000) + let session = makeSession() + let transcoder = try XCTUnwrap(ProAVSpatialAudioTranscoder(stream: stream, session: session)) + defer { transcoder.cancel() } + transcoder.begin(origin: .zero, videoStart: .zero) + var packet = av_packet_alloc() + defer { av_packet_free(&packet) } + let corePacket = try XCTUnwrap(packet) + XCTAssertEqual(av_new_packet(corePacket, 64 * 1024), 0) + memset(corePacket.pointee.data, 0, 64 * 1024) + // 130 packets of 64 KB without a TrueHD major sync pass the 8 MB bound. + for index in 0 ..< 130 { + corePacket.pointee.pts = Int64(index) * 50 + transcoder.append(packet: corePacket) + } + let deadline = Date(timeIntervalSinceNow: 10) + while !session.isFailed, Date() < deadline { + RunLoop.current.run(until: Date(timeIntervalSinceNow: 0.01)) + } + XCTAssertEqual(session.spatialAudioFailure, .failed) + #else + throw XCTSkip("SpatialAudioBridge is unavailable on this platform") + #endif + } + func testReadyWaitsForTheAPACRendition() throws { - let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) - defer { try? FileManager.default.removeItem(at: directory) } - let session = ProAVRemuxSession(configuration: .init(directory: directory)) + let session = makeSession() var readyCount = 0 session.onReady = { _ in readyCount += 1 } - let video = ProAVVideoSignaling(codecTag: "avc1", codecsAttribute: "avc1.640028", videoRange: "SDR", supplementalCodecs: nil, preferredDynamicRange: .sdr) XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) XCTAssertTrue(session.playsSpatialAudio) for start in [0.0, 2, 4] { @@ -61,6 +153,7 @@ class ProAVSpatialAudioTest: XCTestCase { session.appendAudioSegment(Data([1]), duration: 3.2) XCTAssertEqual(readyCount, 1) session.finish(reachedEnd: true) + let directory = session.configuration.directory let playlist = try String(contentsOf: directory.appendingPathComponent(ProAVRemuxSession.audioPlaylistName), encoding: .utf8) XCTAssertTrue(playlist.contains("#EXT-X-MAP:URI=\"audio-init.mp4\"\n#EXTINF:3.20000,\naudio0.m4s\n#EXT-X-ENDLIST\n")) // HLS requires the audio and video playlists of a variant to share one target duration. @@ -69,29 +162,41 @@ class ProAVSpatialAudioTest: XCTestCase { XCTAssertTrue(media.contains("#EXT-X-TARGETDURATION:4\n")) } - func testEndingWithoutAPACFragmentsFallsBackToFLAC() { - let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) - defer { try? FileManager.default.removeItem(at: directory) } - let session = ProAVRemuxSession(configuration: .init(directory: directory)) - let video = ProAVVideoSignaling(codecTag: "avc1", codecsAttribute: "avc1.640028", videoRange: "SDR", supplementalCodecs: nil, preferredDynamicRange: .sdr) + func testEndingWithoutAPACFragmentsFallsBackForTheLaunch() { + let session = makeSession() XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) session.closeSegment(nextStartTime: 0) session.closeSegment(nextStartTime: 2) session.finish(reachedEnd: true) XCTAssertTrue(session.isFailed) - XCTAssertTrue(session.spatialAudioFailed) + XCTAssertEqual(session.spatialAudioFailure, .failed) } func testOnlySpatialAudioFailuresFallBackToFLAC() { - let directory = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) - let failed = ProAVRemuxSession(configuration: .init(directory: directory)) + let failed = makeSession() failed.fail(NSError(description: "segment write failed")) - failed.failSpatialAudio(NSError(description: "unsupported metadata")) + failed.failSpatialAudio(NSError(description: "unsupported metadata"), refusedTrackID: 1) XCTAssertTrue(failed.isFailed) - XCTAssertFalse(failed.spatialAudioFailed) - let rejected = ProAVRemuxSession(configuration: .init(directory: directory)) - rejected.failSpatialAudio(NSError(description: "unsupported metadata")) - XCTAssertTrue(rejected.isFailed) - XCTAssertTrue(rejected.spatialAudioFailed) + XCTAssertNil(failed.spatialAudioFailure) + let refused = makeSession() + refused.failSpatialAudio(NSError(description: "unsupported metadata"), refusedTrackID: 1) + XCTAssertTrue(refused.isFailed) + XCTAssertEqual(refused.spatialAudioFailure, .refused(trackID: 1)) + } + + func testRefusedTracksKeepFLACWhileOthersTryAPAC() { + var policy = ProAVRemuxSession.SpatialAudioPolicy(refusedTrackIDs: [1]) + XCTAssertFalse(policy.allows(trackID: 1)) + XCTAssertTrue(policy.allows(trackID: 2)) + policy.isEnabled = false + XCTAssertFalse(policy.allows(trackID: 2)) + } + + func testRemuxMeasuresHowFarItReadsAhead() { + let session = makeSession() + session.trackVideoTime(seconds: 640) + XCTAssertNil(session.remuxedAhead) + session.playbackTime = 600 + XCTAssertEqual(session.remuxedAhead, 40) } } From 0b518940ccdd0912888189a9b139f1a041d8ffef Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:04:35 +0200 Subject: [PATCH 17/36] docs: note the license notices, untested ios and the read-ahead --- README.md | 4 +++- docs/03-engine-meplayer-demux-e-pipeline.md | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 9d66568..2dfcc15 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- **TrueHD Atmos and DTS:X play as APAC** on tvOS, iOS and macOS 26 and later: [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen +- **TrueHD Atmos and DTS:X play as APAC** on tvOS and macOS 26 and later (iOS builds, untested): [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen - Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode @@ -192,3 +192,5 @@ Lumen grew out of [**KSPlayer**](https://github.com/kingslay/KSPlayer) by [kings ## License GPL-3.0 — see [`LICENSE`](./LICENSE). + +Apps that ship Lumen show `SpatialAudioBridgeNotices.text`, the license notices of SpatialAudioBridge's core, in their acknowledgements. diff --git a/docs/03-engine-meplayer-demux-e-pipeline.md b/docs/03-engine-meplayer-demux-e-pipeline.md index 81109c2..1946ad6 100644 --- a/docs/03-engine-meplayer-demux-e-pipeline.md +++ b/docs/03-engine-meplayer-demux-e-pipeline.md @@ -144,7 +144,7 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa Áudio, por `ProAVAudioStrategy`: EC-3, AC-3, AAC, FLAC e ALAC passam por **stream copy**; qualquer outro codec (DTS, TrueHD, PCM, Opus…) é **transcodificado para FLAC** por `ProAVAudioTranscoder` (S16 ou S32/24 bits, sample rate e layout de canais preservados, PTS regenerado por contagem de amostras). Se o transcoder não puder inicializar, a trilha de áudio é simplesmente descartada. -No tvOS, iOS e macOS 26 ou mais novos, TrueHD e DTS-HD MA a 48 kHz tentam primeiro o **APAC**: `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Trilhas sem Atmos ou DTS:X, ou com metadata que a biblioteca não renderiza, falham a sessão assim que a decodificação começa (`spatialAudioFailed`); o `ProAVPlayer` então reinicia o launch na posição atual pelo caminho FLAC, que vale para o resto do item ou até outra trilha ser escolhida. Enquanto o APAC toca, um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. +No tvOS, iOS e macOS 26 ou mais novos, TrueHD com o perfil Atmos do FFmpeg e todo DTS-HD MA a 48 kHz tentam primeiro o **APAC** (o FFmpeg rotula algumas apresentações DTS:X como DTS-HD MA simples): `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Quando a conversão falha, a sessão falha com `spatialAudioFailure` e o `ProAVPlayer` reinicia o launch na posição atual pelo caminho FLAC. Só uma recusa de decodificação da biblioteca (trilha sem Atmos ou DTS:X, ou com metadata que ela não renderiza) fixa o FLAC para aquela trilha da fonte; outras falhas valem só para aquele launch, e o próximo seek tenta o APAC de novo. Enquanto o APAC toca, um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. Nos demais casos, a trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. @@ -164,7 +164,7 @@ Perfil 7 é dual-layer (base layer + enhancement layer + RPU) e não tem sample - **Sem legendas embutidas**: o remux carrega apenas vídeo e uma trilha de áudio; legendas do container não entram na playlist. - **A sinalização de Atmos depende do primeiro packet de áudio**: o `dec3` só sai completo porque o `moov` é adiado (`+delay_moov`) até o muxer ter parseado um frame E-AC-3. Se a trilha mapeada não entregar um packet com sync word dentro da janela do primeiro segmento, a sessão falha de propósito — antes ela emitiria um `dec3` zerado. O `CHANNELS="16/JOC"` da playlist, por outro lado, vem do `codecpar.profile` (decoder), não do muxer: as duas metades da sinalização têm fontes independentes e podem, em tese, discordar. - **RPU só é manipulada no perfil 7**: em DV perfil 5 e 8 as RPUs seguem embutidas no bitstream HEVC, sem extração nem reinjeção, e quem as interpreta é o decoder da Apple. A única reescrita é a conversão de perfil 7 para 8.1 descrita acima. -- **Consumo de disco**: o workspace cresce enquanto o remux avança, e só é limpo na próxima inicialização do player. `switchSource` marca o diretório do launch descartado para remoção (`requestCleanup`), que acontece quando aquele remux termina de fechar. +- **Consumo de disco**: o workspace cresce enquanto o remux avança, e só é limpo na próxima inicialização do player. O remux lê no máximo `maxBufferDuration` à frente da reprodução, como o engine de decodificação: pausa acima disso e retoma abaixo da metade, com a posição vinda de um observer periódico do `AVPlayer`. `switchSource` marca o diretório do launch descartado para remoção (`requestCleanup`), que acontece quando aquele remux termina de fechar. ## Cache de disco por byte-range From ed2e1d12311951c0178d689f8f2ed848c44bc41a Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:43:59 +0200 Subject: [PATCH 18/36] build: move to spatialaudiobridge 1.5.0 1.5.0 ships universal macOS and iOS simulator slices, so universal Mac builds of apps that embed Lumen work again; generic Apple TV simulator builds still need ARCHS=arm64. The README now says so, and lists iOS as building with spatial audio on AirPods still to be tested. --- Package.swift | 4 ++-- README.md | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Package.swift b/Package.swift index 667fdf3..6bafb27 100644 --- a/Package.swift +++ b/Package.swift @@ -21,7 +21,7 @@ let package = Package( name: "Lumen", dependencies: [ .product(name: "FFmpegKit", package: "FFmpegKit"), - // The core ships arm64 slices for Apple TV, iPhone, iPad and Mac, without Mac Catalyst. + // The core has no Mac Catalyst slices. .product(name: "SpatialAudioBridge", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), // .product(name: "Libass", package: "FFmpegKit"), @@ -46,5 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), - .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.3.0"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.5.0"), ] diff --git a/README.md b/README.md index 2dfcc15..e429c89 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- **TrueHD Atmos and DTS:X play as APAC** on tvOS and macOS 26 and later (iOS builds, untested): [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen +- **TrueHD Atmos and DTS:X play as APAC** on tvOS and macOS 26 and later (iOS: builds; spatial audio on AirPods to be tested): [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen - Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode @@ -82,7 +82,7 @@ There are two floors, and they are not the same one: Scrub thumbnails are tvOS-only, at any version. - Xcode 26 or later: SpatialAudioBridge, which converts TrueHD Atmos and DTS:X, needs Swift 6 tools and the tvOS, iOS and macOS 26 SDKs to build. Lumen itself still compiles in the Swift 5 language mode -- SpatialAudioBridge's core ships arm64 slices only: build macOS apps for Apple silicon (`ARCHS=arm64`). Mac Catalyst builds leave it out and keep the FLAC path +- SpatialAudioBridge's core is universal for macOS and the iOS simulator; generic Apple TV simulator builds need `ARCHS=arm64`. Mac Catalyst builds leave it out and keep the FLAC path ## Installation From 05feefb9f114be9744835721201d3bb7ab647652 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 03:43:59 +0200 Subject: [PATCH 19/36] fix: pace the remux on the file's timeline, share the multichannel hold, skip apac on airplay - The read-ahead compared the source's own video times with a playback position counted from the start of the file, so a file whose times start late, such as a transport stream, paused the remux for good. The item now subtracts the format's start time. - Each launch seeds the playback position with its start, so a cold start, a restart or a source-switch candidate reads at most the buffer duration ahead before the player reports a position. - Multichannel content is a process-wide setting, so it is now held while any player plays APAC and returns to its previous value after the last. - An AirPlay receiver plays the stream itself, so launches leave the APAC rendition out while external playback is active, and a launch playing APAC restarts on FLAC when AirPlay starts. --- Sources/Lumen/AVPlayer/KSOptions.swift | 24 +++++++++++++--- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 2 +- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 28 +++++++++++++++++-- .../Lumen/MEPlayer/ProAVRemuxSession.swift | 9 +++--- Tests/LumenTests/ProAVSpatialAudioTest.swift | 7 +++-- 5 files changed, 56 insertions(+), 14 deletions(-) diff --git a/Sources/Lumen/AVPlayer/KSOptions.swift b/Sources/Lumen/AVPlayer/KSOptions.swift index 9601bba..c23ce90 100644 --- a/Sources/Lumen/AVPlayer/KSOptions.swift +++ b/Sources/Lumen/AVPlayer/KSOptions.swift @@ -555,12 +555,28 @@ public extension KSOptions { #endif } - /// Multichannel content is on while the remux plays APAC, as the Atmos output was tested. - static func setSupportsMultichannelContent(_ supports: Bool) { + // Only touched on the audio session queue. + private nonisolated(unsafe) static var multichannelContentHolders = 0 + private nonisolated(unsafe) static var multichannelContentBeforeHolders = false + + /// Multichannel content is on while any remux plays APAC, as the Atmos output was tested, and + /// returns to its previous setting when the last one stops. + static func holdMultichannelContent(_ hold: Bool) { #if !os(macOS) audioSessionQueue.async { - if #available(tvOS 15.0, iOS 15.0, *) { - try? AVAudioSession.sharedInstance().setSupportsMultichannelContent(supports) + guard #available(tvOS 15.0, iOS 15.0, *) else { return } + let session = AVAudioSession.sharedInstance() + if hold { + if multichannelContentHolders == 0 { + multichannelContentBeforeHolders = session.supportsMultichannelContent + try? session.setSupportsMultichannelContent(true) + } + multichannelContentHolders += 1 + } else if multichannelContentHolders > 0 { + multichannelContentHolders -= 1 + if multichannelContentHolders == 0 { + try? session.setSupportsMultichannelContent(multichannelContentBeforeHolders) + } } } #endif diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 7bc15c3..3437eed 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -1100,7 +1100,7 @@ extension MEPlayerItem: CodecCapacityDelegate { func codecDidChangeCapacity() { if let remuxSession { // The remux reads ahead of playback by the buffer duration, as the decoding path buffers. - if let ahead = remuxSession.remuxedAhead { + if let ahead = remuxSession.remuxedAhead(sourceStart: startTime.seconds) { if ahead > options.maxBufferDuration { pause() } else if ahead < options.maxBufferDuration / 2 { diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index d577901..a95150d 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -38,11 +38,12 @@ public final class ProAVPlayer { /// Whether playback resumes after the APAC fallback restarts a launch that was already playing. private var fallbackResumesPlayback: Bool? private var playbackTimeObserver: Any? + private var externalPlaybackObservation: NSKeyValueObservation? /// The AtmosOutputActivator while a launch plays APAC; releasing it detaches it. private var atmosOutput: AnyObject? { didSet { if (atmosOutput == nil) != (oldValue == nil) { - KSOptions.setSupportsMultichannelContent(atmosOutput != nil) + KSOptions.holdMultichannelContent(atmosOutput != nil) } } } @@ -83,6 +84,14 @@ public final class ProAVPlayer { self.session?.playbackTime = self.currentPlaybackTime } } + #if !os(xrOS) + externalPlaybackObservation = innerPlayer.player.observe(\.isExternalPlaybackActive) { [weak self] player, _ in + let isActive = player.isExternalPlaybackActive + Task { @MainActor in + self?.externalPlaybackDidChange(isActive: isActive) + } + } + #endif } private static var workspaceRoot: URL { @@ -104,7 +113,11 @@ public final class ProAVPlayer { let configuration = ProAVRemuxSession.Configuration(directory: directory, targetSegmentDuration: ProAVPlayer.segmentDuration, minimumSegmentsBeforeReady: ProAVPlayer.minimumSegmentsBeforeReady) let session = ProAVRemuxSession(configuration: configuration) session.preferredAudioTrackID = audioTrackID + var spatialAudio = spatialAudio + // An AirPlay receiver plays the stream itself, so the APAC rendition is left out. + spatialAudio.isEnabled = spatialAudio.isEnabled && !innerPlayer.isExternalPlaybackActive session.spatialAudioPolicy = spatialAudio + session.playbackTime = time options.startPlayTime = time let item = MEPlayerItem(url: url, options: options, remuxSession: session) let proxy = ProAVLaunchDelegateProxy(target: self, launch: launchIndex) @@ -203,9 +216,20 @@ public final class ProAVPlayer { if case let .refused(trackID) = failure { refusedSpatialAudioTrackIDs.insert(trackID) } + restartWithoutSpatialAudio() + } + + private func externalPlaybackDidChange(isActive: Bool) { + guard isActive, !didFail, session?.playsSpatialAudio == true else { return } + KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") + restartWithoutSpatialAudio() + } + + /// Restarts the active launch at the current position on the FLAC path. + private func restartWithoutSpatialAudio() { let seekCompletion = pendingSeekCompletion pendingSeekCompletion = nil - // The remux runs ahead of playback, so the failure can arrive while this launch plays. + // The remux runs ahead of playback, so this can happen while the launch plays. let resumesPlayback = isReadyToPlay && seekCompletion == nil ? isPlaying : nil restart(at: currentPlaybackTime, completion: seekCompletion, spatialAudioEnabled: false) fallbackResumesPlayback = resumesPlayback diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 823e106..7c18b25 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -108,7 +108,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { _spatialAudioFailure } } - /// The player's position, in source time, which the remux reads ahead of. + /// The player's position from the start of the file, which the remux reads ahead of. var playbackTime: TimeInterval? { get { withLock { _playbackTime } @@ -118,11 +118,12 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } - /// How far the remuxed video runs ahead of playback, once both are known. - var remuxedAhead: TimeInterval? { + /// How far the remuxed video runs ahead of playback, once both are known. Video times are the + /// source's own, which start at `sourceStart`. + func remuxedAhead(sourceStart: TimeInterval) -> TimeInterval? { withLock { guard let lastVideoSeconds, let _playbackTime else { return nil } - return lastVideoSeconds - _playbackTime + return lastVideoSeconds - sourceStart - _playbackTime } } diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index 6cc9a91..4345659 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -194,9 +194,10 @@ class ProAVSpatialAudioTest: XCTestCase { func testRemuxMeasuresHowFarItReadsAhead() { let session = makeSession() - session.trackVideoTime(seconds: 640) - XCTAssertNil(session.remuxedAhead) + session.trackVideoTime(seconds: 641.4) + XCTAssertNil(session.remuxedAhead(sourceStart: 1.4)) + // Playback counts from the start of the file; a transport stream's own times start later. session.playbackTime = 600 - XCTAssertEqual(session.remuxedAhead, 40) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? 0, 40, accuracy: 0.001) } } From b2417e339fc23ae3bb654e6457fd059e90c016e2 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:15:37 +0200 Subject: [PATCH 20/36] fix: keep the atmos activator to tvos and tie up the multichannel hold and airplay - The silent Dolby Digital Plus Atmos stream exists to switch an HDMI receiver into Atmos, so the activator now runs on tvOS only and SpatialAudioPlayback is linked there alone. iOS and macOS still hold multichannel content while a launch plays APAC. - A player released without a shutdown gives up its multichannel hold. - When AirPlay starts while a source-switch candidate is preparing an APAC rendition, the candidate restarts without it. --- Package.swift | 4 +- README.md | 2 +- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 57 +++++++++++++++------ docs/03-engine-meplayer-demux-e-pipeline.md | 2 +- 4 files changed, 46 insertions(+), 19 deletions(-) diff --git a/Package.swift b/Package.swift index 6bafb27..0bd562e 100644 --- a/Package.swift +++ b/Package.swift @@ -21,9 +21,9 @@ let package = Package( name: "Lumen", dependencies: [ .product(name: "FFmpegKit", package: "FFmpegKit"), - // The core has no Mac Catalyst slices. + // The core has no Mac Catalyst slices; the Atmos activator is for Apple TV's HDMI output. .product(name: "SpatialAudioBridge", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), - .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), + .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS])), // .product(name: "Libass", package: "FFmpegKit"), // .product(name: "Libmpv", package: "FFmpegKit"), "DisplayCriteria", diff --git a/README.md b/README.md index e429c89..ba2e908 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- **TrueHD Atmos and DTS:X play as APAC** on tvOS and macOS 26 and later (iOS: builds; spatial audio on AirPods to be tested): [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen +- **TrueHD Atmos and DTS:X play as APAC** on tvOS and macOS 26 and later (iOS: builds; spatial audio on AirPods to be tested): [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and on tvOS a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen - Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index a95150d..71fcf32 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -3,7 +3,7 @@ import Foundation #if os(tvOS) || os(xrOS) import DisplayCriteria #endif -#if canImport(SpatialAudioPlayback) +#if os(tvOS) && canImport(SpatialAudioPlayback) import SpatialAudioPlayback #endif #if canImport(UIKit) @@ -39,11 +39,12 @@ public final class ProAVPlayer { private var fallbackResumesPlayback: Bool? private var playbackTimeObserver: Any? private var externalPlaybackObservation: NSKeyValueObservation? - /// The AtmosOutputActivator while a launch plays APAC; releasing it detaches it. - private var atmosOutput: AnyObject? { + /// The AtmosOutputActivator while a launch plays APAC on tvOS; releasing it detaches it. + private var atmosOutput: AnyObject? + private var holdsMultichannelContent = false { didSet { - if (atmosOutput == nil) != (oldValue == nil) { - KSOptions.holdMultichannelContent(atmosOutput != nil) + if holdsMultichannelContent != oldValue { + KSOptions.holdMultichannelContent(holdsMultichannelContent) } } } @@ -94,6 +95,13 @@ public final class ProAVPlayer { #endif } + deinit { + // A player released without a shutdown still gives up its hold. + if holdsMultichannelContent { + KSOptions.holdMultichannelContent(false) + } + } + private static var workspaceRoot: URL { let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first ?? FileManager.default.temporaryDirectory return base.appendingPathComponent("Lumen-ProAV") @@ -172,16 +180,19 @@ public final class ProAVPlayer { private func attach(baseURL: URL, directoryName: String) { guard !didFail else { return } let masterURL = baseURL.appendingPathComponent(directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) - updateAtmosOutput() + updateSpatialAudioOutput() innerPlayer.replace(url: masterURL, options: options) innerPlayer.prepareToPlay() } - /// Keeps the receiver in Dolby Atmos while the active launch plays APAC, which alone does not switch it. - private func updateAtmosOutput() { - #if canImport(SpatialAudioPlayback) - guard #available(macOS 26, iOS 26, tvOS 26, *) else { return } - guard session?.playsSpatialAudio == true else { + /// Holds multichannel content while the active launch plays APAC, and on tvOS keeps the receiver + /// in Dolby Atmos, which APAC alone does not switch. + private func updateSpatialAudioOutput() { + let playsSpatialAudio = session?.playsSpatialAudio == true + holdsMultichannelContent = playsSpatialAudio + #if os(tvOS) && canImport(SpatialAudioPlayback) + guard #available(tvOS 26, *) else { return } + guard playsSpatialAudio else { atmosOutput = nil return } @@ -192,6 +203,11 @@ public final class ProAVPlayer { #endif } + private func stopSpatialAudioOutput() { + atmosOutput = nil + holdsMultichannelContent = false + } + private func applyDisplayCriteria() { #if os(tvOS) || os(xrOS) guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager, @@ -220,7 +236,13 @@ public final class ProAVPlayer { } private func externalPlaybackDidChange(isActive: Bool) { - guard isActive, !didFail, session?.playsSpatialAudio == true else { return } + guard isActive, !didFail else { return } + if let pending = pendingSourceSwitch, pending.spatialAudio.isEnabled { + var spatialAudio = pending.spatialAudio + spatialAudio.isEnabled = false + restartPendingSourceSwitch(spatialAudio: spatialAudio) + } + guard session?.playsSpatialAudio == true else { return } KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") restartWithoutSpatialAudio() } @@ -243,7 +265,7 @@ public final class ProAVPlayer { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - atmosOutput = nil + stopSpatialAudioOutput() innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -303,6 +325,11 @@ public final class ProAVPlayer { if case let .refused(trackID) = failure { spatialAudio.refusedTrackIDs.insert(trackID) } + restartPendingSourceSwitch(spatialAudio: spatialAudio) + } + + private func restartPendingSourceSwitch(spatialAudio: ProAVRemuxSession.SpatialAudioPolicy) { + guard let pending = pendingSourceSwitch else { return } pendingSourceSwitch = nil if pending.innerSwitchStarted { innerPlayer.abandonPendingSourceSwitch() @@ -355,7 +382,7 @@ public final class ProAVPlayer { if session?.videoSignaling?.preferredDynamicRange != previousDynamicRange { applyDisplayCriteria() } - updateAtmosOutput() + updateSpatialAudioOutput() pending.completion(true) } @@ -483,7 +510,7 @@ extension ProAVPlayer: MediaPlayerProtocol { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - atmosOutput = nil + stopSpatialAudioOutput() innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() diff --git a/docs/03-engine-meplayer-demux-e-pipeline.md b/docs/03-engine-meplayer-demux-e-pipeline.md index 1946ad6..740fd90 100644 --- a/docs/03-engine-meplayer-demux-e-pipeline.md +++ b/docs/03-engine-meplayer-demux-e-pipeline.md @@ -144,7 +144,7 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa Áudio, por `ProAVAudioStrategy`: EC-3, AC-3, AAC, FLAC e ALAC passam por **stream copy**; qualquer outro codec (DTS, TrueHD, PCM, Opus…) é **transcodificado para FLAC** por `ProAVAudioTranscoder` (S16 ou S32/24 bits, sample rate e layout de canais preservados, PTS regenerado por contagem de amostras). Se o transcoder não puder inicializar, a trilha de áudio é simplesmente descartada. -No tvOS, iOS e macOS 26 ou mais novos, TrueHD com o perfil Atmos do FFmpeg e todo DTS-HD MA a 48 kHz tentam primeiro o **APAC** (o FFmpeg rotula algumas apresentações DTS:X como DTS-HD MA simples): `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Quando a conversão falha, a sessão falha com `spatialAudioFailure` e o `ProAVPlayer` reinicia o launch na posição atual pelo caminho FLAC. Só uma recusa de decodificação da biblioteca (trilha sem Atmos ou DTS:X, ou com metadata que ela não renderiza) fixa o FLAC para aquela trilha da fonte; outras falhas valem só para aquele launch, e o próximo seek tenta o APAC de novo. Enquanto o APAC toca, um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. +No tvOS, iOS e macOS 26 ou mais novos, TrueHD com o perfil Atmos do FFmpeg e todo DTS-HD MA a 48 kHz tentam primeiro o **APAC** (o FFmpeg rotula algumas apresentações DTS:X como DTS-HD MA simples): `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Quando a conversão falha, a sessão falha com `spatialAudioFailure` e o `ProAVPlayer` reinicia o launch na posição atual pelo caminho FLAC. Só uma recusa de decodificação da biblioteca (trilha sem Atmos ou DTS:X, ou com metadata que ela não renderiza) fixa o FLAC para aquela trilha da fonte; outras falhas valem só para aquele launch, e o próximo seek tenta o APAC de novo. No tvOS, enquanto o APAC toca, um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. Nos demais casos, a trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. From 3132e3076ce343fe423975629eb9d16d1469a27c Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:27:27 +0200 Subject: [PATCH 21/36] fix: pace the remux only once its launch is ready A launch that needed more than the buffer duration of source to reach the readiness gate, such as one with long GOPs, paused before the player attached, and its seeded playback position never moved, so it stalled for good. The read-ahead now applies once the session has fired ready. --- Sources/Lumen/MEPlayer/ProAVRemuxSession.swift | 7 ++++--- Tests/LumenTests/ProAVSpatialAudioTest.swift | 8 ++++++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 7c18b25..d906b8e 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -118,11 +118,12 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } - /// How far the remuxed video runs ahead of playback, once both are known. Video times are the - /// source's own, which start at `sourceStart`. + /// How far the remuxed video runs ahead of playback, once the session is ready and both are known, + /// so the remux always reaches its readiness gate. Video times are the source's own, which start + /// at `sourceStart`. func remuxedAhead(sourceStart: TimeInterval) -> TimeInterval? { withLock { - guard let lastVideoSeconds, let _playbackTime else { return nil } + guard readyFired, let lastVideoSeconds, let _playbackTime else { return nil } return lastVideoSeconds - sourceStart - _playbackTime } } diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index 4345659..0d6200b 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -192,12 +192,16 @@ class ProAVSpatialAudioTest: XCTestCase { XCTAssertFalse(policy.allows(trackID: 2)) } - func testRemuxMeasuresHowFarItReadsAhead() { + func testRemuxMeasuresHowFarItReadsAheadOnceReady() { let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: nil, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) session.trackVideoTime(seconds: 641.4) - XCTAssertNil(session.remuxedAhead(sourceStart: 1.4)) // Playback counts from the start of the file; a transport stream's own times start later. session.playbackTime = 600 + XCTAssertNil(session.remuxedAhead(sourceStart: 1.4)) + for start in [600.0, 602, 604] { + session.closeSegment(nextStartTime: start) + } XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? 0, 40, accuracy: 0.001) } } From 9658aad0103b3a4926a32d349b434e62d09ac84b Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:36:24 +0200 Subject: [PATCH 22/36] fix: rebuild a switch candidate for airplay only when the active launch stays When AirPlay started while both the active launch and a switch candidate played APAC, the candidate was rebuilt and then abandoned by the active launch's restart, as any restart abandons a pending switch. The active launch now restarts on its own in that case, and the layer completes the switch cold, without APAC while AirPlay lasts. --- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 71fcf32..1134291 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -237,14 +237,16 @@ public final class ProAVPlayer { private func externalPlaybackDidChange(isActive: Bool) { guard isActive, !didFail else { return } - if let pending = pendingSourceSwitch, pending.spatialAudio.isEnabled { + if session?.playsSpatialAudio == true { + KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") + // Like a seek, the restart abandons a pending switch, which the layer then makes cold, + // without APAC while AirPlay lasts. + restartWithoutSpatialAudio() + } else if let pending = pendingSourceSwitch, pending.spatialAudio.isEnabled { var spatialAudio = pending.spatialAudio spatialAudio.isEnabled = false restartPendingSourceSwitch(spatialAudio: spatialAudio) } - guard session?.playsSpatialAudio == true else { return } - KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") - restartWithoutSpatialAudio() } /// Restarts the active launch at the current position on the FLAC path. From 3090903dd7d42cb271a9d601a3aeeb5dc3a3b68c Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 04:48:15 +0200 Subject: [PATCH 23/36] fix: bring apac back when airplay ends A launch restarted on FLAC for AirPlay, or started during it, stayed on FLAC after playback returned to the device. The session now records when it left APAC out for an eligible track the source has not refused, and the player restarts such a launch in place with APAC once external playback ends. --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 4 +++- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 22 +++++++++++++------ .../Lumen/MEPlayer/ProAVRemuxSession.swift | 12 ++++++++++ .../ProAVSpatialAudioTranscoder.swift | 10 +++++++++ 4 files changed, 40 insertions(+), 8 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 3437eed..dc82864 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -358,7 +358,9 @@ extension MEPlayerItem { let audioAssetTrack = audios.first { preferredAudioTrackID == nil ? $0.isEnabled : $0.trackID == preferredAudioTrackID } ?? audios.first { $0.isEnabled } ?? audios.first if let audioAssetTrack, let inputAudioStream = formatCtx.pointee.streams[Int(audioAssetTrack.trackID)] { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) - if session.spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { + let spatialAudioPolicy = session.spatialAudioPolicy + session.spatialAudioWithheld = !spatialAudioPolicy.isEnabled && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) + if spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { remuxSpatialAudio = spatialAudio audioSignaling = spatialAudio.signaling } else if strategy.copiesBitstream { diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 1134291..62925fe 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -35,7 +35,8 @@ public final class ProAVPlayer { private var preferredAudioTrackID: Int32? /// Audio tracks of this source that the library refused; they keep the FLAC path. private var refusedSpatialAudioTrackIDs = Set() - /// Whether playback resumes after the APAC fallback restarts a launch that was already playing. + /// Whether playback resumes after an in-place restart, for the FLAC fallback or AirPlay, of a launch + /// that was already playing. private var fallbackResumesPlayback: Bool? private var playbackTimeObserver: Any? private var externalPlaybackObservation: NSKeyValueObservation? @@ -232,16 +233,23 @@ public final class ProAVPlayer { if case let .refused(trackID) = failure { refusedSpatialAudioTrackIDs.insert(trackID) } - restartWithoutSpatialAudio() + restartInPlace(spatialAudioEnabled: false) } private func externalPlaybackDidChange(isActive: Bool) { - guard isActive, !didFail else { return } + guard !didFail else { return } + guard isActive else { + // Back on this device, a launch that left APAC out for AirPlay brings it back. + if session?.spatialAudioWithheld == true { + restartInPlace(spatialAudioEnabled: true) + } + return + } if session?.playsSpatialAudio == true { KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") // Like a seek, the restart abandons a pending switch, which the layer then makes cold, // without APAC while AirPlay lasts. - restartWithoutSpatialAudio() + restartInPlace(spatialAudioEnabled: false) } else if let pending = pendingSourceSwitch, pending.spatialAudio.isEnabled { var spatialAudio = pending.spatialAudio spatialAudio.isEnabled = false @@ -249,13 +257,13 @@ public final class ProAVPlayer { } } - /// Restarts the active launch at the current position on the FLAC path. - private func restartWithoutSpatialAudio() { + /// Restarts the active launch at the current position, keeping the play or pause state. + private func restartInPlace(spatialAudioEnabled: Bool) { let seekCompletion = pendingSeekCompletion pendingSeekCompletion = nil // The remux runs ahead of playback, so this can happen while the launch plays. let resumesPlayback = isReadyToPlay && seekCompletion == nil ? isPlaying : nil - restart(at: currentPlaybackTime, completion: seekCompletion, spatialAudioEnabled: false) + restart(at: currentPlaybackTime, completion: seekCompletion, spatialAudioEnabled: spatialAudioEnabled) fallbackResumesPlayback = resumesPlayback } diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index d906b8e..aa41315 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -45,6 +45,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { private var _preferredAudioTrackID: Int32? private var _spatialAudioPolicy = SpatialAudioPolicy() private var _spatialAudioFailure: SpatialAudioFailure? + private var _spatialAudioWithheld = false private var _playbackTime: TimeInterval? private var _onReady: ((URL) -> Void)? private var _onFailure: ((NSError) -> Void)? @@ -108,6 +109,17 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { _spatialAudioFailure } } + /// Whether this launch left APAC out for an eligible track that its source has not refused, as during + /// AirPlay, so a later launch can bring it back. + var spatialAudioWithheld: Bool { + get { + withLock { _spatialAudioWithheld } + } + set { + withLock { _spatialAudioWithheld = newValue } + } + } + /// The player's position from the start of the file, which the remux reads ahead of. var playbackTime: TimeInterval? { get { diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift index 34c7029..01b4a70 100644 --- a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -103,6 +103,16 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { queue.finish() } + /// Whether a track would convert on this system, given the chance. + static func converts(codecpar: AVCodecParameters) -> Bool { + #if canImport(SpatialAudioBridge) + if #available(macOS 26, iOS 26, tvOS 26, *) { + return codec(codecpar: codecpar) != nil + } + #endif + return false + } + #if canImport(SpatialAudioBridge) @available(macOS 26, iOS 26, tvOS 26, *) static func codec(codecpar: AVCodecParameters) -> SpatialAudioCodec? { From d61822a71a665e8d229f500a39399ffdcd7688bd Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 13:29:06 +0200 Subject: [PATCH 24/36] fix: drop the multichannel hold, which a seek undid and atmos does not need Every new remux item runs KSOptions.isSpatialAudioEnabled for its audio tracks, which turns multichannel content off on an HDMI route. A seek keeps the player's hold set, so nothing turned it back on. The receiver still showed Atmos with it off; on tvOS the activator is what switches it, and on iOS the existing route check already turns it on for spatial outputs. --- Sources/Lumen/AVPlayer/KSOptions.swift | 27 --------------------- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 31 ++++-------------------- 2 files changed, 5 insertions(+), 53 deletions(-) diff --git a/Sources/Lumen/AVPlayer/KSOptions.swift b/Sources/Lumen/AVPlayer/KSOptions.swift index c23ce90..136ca43 100644 --- a/Sources/Lumen/AVPlayer/KSOptions.swift +++ b/Sources/Lumen/AVPlayer/KSOptions.swift @@ -555,33 +555,6 @@ public extension KSOptions { #endif } - // Only touched on the audio session queue. - private nonisolated(unsafe) static var multichannelContentHolders = 0 - private nonisolated(unsafe) static var multichannelContentBeforeHolders = false - - /// Multichannel content is on while any remux plays APAC, as the Atmos output was tested, and - /// returns to its previous setting when the last one stops. - static func holdMultichannelContent(_ hold: Bool) { - #if !os(macOS) - audioSessionQueue.async { - guard #available(tvOS 15.0, iOS 15.0, *) else { return } - let session = AVAudioSession.sharedInstance() - if hold { - if multichannelContentHolders == 0 { - multichannelContentBeforeHolders = session.supportsMultichannelContent - try? session.setSupportsMultichannelContent(true) - } - multichannelContentHolders += 1 - } else if multichannelContentHolders > 0 { - multichannelContentHolders -= 1 - if multichannelContentHolders == 0 { - try? session.setSupportsMultichannelContent(multichannelContentBeforeHolders) - } - } - } - #endif - } - #if !os(macOS) static func isSpatialAudioEnabled(channelCount _: AVAudioChannelCount) -> Bool { if #available(tvOS 15.0, iOS 15.0, *) { diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 62925fe..ce6748b 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -42,13 +42,6 @@ public final class ProAVPlayer { private var externalPlaybackObservation: NSKeyValueObservation? /// The AtmosOutputActivator while a launch plays APAC on tvOS; releasing it detaches it. private var atmosOutput: AnyObject? - private var holdsMultichannelContent = false { - didSet { - if holdsMultichannelContent != oldValue { - KSOptions.holdMultichannelContent(holdsMultichannelContent) - } - } - } private var reportedReady = false private var didFail = false public weak var delegate: MediaPlayerDelegate? @@ -96,13 +89,6 @@ public final class ProAVPlayer { #endif } - deinit { - // A player released without a shutdown still gives up its hold. - if holdsMultichannelContent { - KSOptions.holdMultichannelContent(false) - } - } - private static var workspaceRoot: URL { let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first ?? FileManager.default.temporaryDirectory return base.appendingPathComponent("Lumen-ProAV") @@ -186,14 +172,12 @@ public final class ProAVPlayer { innerPlayer.prepareToPlay() } - /// Holds multichannel content while the active launch plays APAC, and on tvOS keeps the receiver - /// in Dolby Atmos, which APAC alone does not switch. + /// On tvOS keeps the receiver in Dolby Atmos while the active launch plays APAC, which APAC alone + /// does not switch. private func updateSpatialAudioOutput() { - let playsSpatialAudio = session?.playsSpatialAudio == true - holdsMultichannelContent = playsSpatialAudio #if os(tvOS) && canImport(SpatialAudioPlayback) guard #available(tvOS 26, *) else { return } - guard playsSpatialAudio else { + guard session?.playsSpatialAudio == true else { atmosOutput = nil return } @@ -204,11 +188,6 @@ public final class ProAVPlayer { #endif } - private func stopSpatialAudioOutput() { - atmosOutput = nil - holdsMultichannelContent = false - } - private func applyDisplayCriteria() { #if os(tvOS) || os(xrOS) guard let displayManager = UIApplication.shared.windows.first?.avDisplayManager, @@ -275,7 +254,7 @@ public final class ProAVPlayer { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - stopSpatialAudioOutput() + atmosOutput = nil innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -520,7 +499,7 @@ extension ProAVPlayer: MediaPlayerProtocol { pendingSeekCompletion = nil seekCompletion?(false) remuxItem?.delegate = nil - stopSpatialAudioOutput() + atmosOutput = nil innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() From 81456acec064e619553f4262616902d1ca9628da Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 14:53:02 +0200 Subject: [PATCH 25/36] feat: play the atmos activator on ios too With AirPods, APAC sounded spatial with head tracking while the silent Atmos stream played and flatter without it, so iOS 26 attaches the activator with the same lifecycle as tvOS. It stops as soon as AirPlay starts, ahead of the restart without APAC. --- Package.swift | 2 +- README.md | 2 +- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 16 +++++++++------- docs/03-engine-meplayer-demux-e-pipeline.md | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/Package.swift b/Package.swift index 0bd562e..111068f 100644 --- a/Package.swift +++ b/Package.swift @@ -23,7 +23,7 @@ let package = Package( .product(name: "FFmpegKit", package: "FFmpegKit"), // The core has no Mac Catalyst slices; the Atmos activator is for Apple TV's HDMI output. .product(name: "SpatialAudioBridge", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS, .macOS])), - .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS])), + .product(name: "SpatialAudioPlayback", package: "SpatialAudioBridge", condition: .when(platforms: [.tvOS, .iOS])), // .product(name: "Libass", package: "FFmpegKit"), // .product(name: "Libmpv", package: "FFmpegKit"), "DisplayCriteria", diff --git a/README.md b/README.md index ba2e908..9cce48d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- **TrueHD Atmos and DTS:X play as APAC** on tvOS and macOS 26 and later (iOS: builds; spatial audio on AirPods to be tested): [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and on tvOS a silent Dolby Digital Plus Atmos stream alongside switches the receiver to Atmos. Not yet checked against a receiver through Lumen +- **TrueHD Atmos and DTS:X play as APAC** on tvOS, iOS and macOS 26 and later: [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and on tvOS and iOS a silent Dolby Digital Plus Atmos stream plays alongside, which switches a receiver to Atmos. A receiver showed Atmos through Lumen on Apple TV; spatial audio on AirPods is still to be tested - Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index ce6748b..8649ad6 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -3,7 +3,7 @@ import Foundation #if os(tvOS) || os(xrOS) import DisplayCriteria #endif -#if os(tvOS) && canImport(SpatialAudioPlayback) +#if (os(tvOS) || os(iOS)) && canImport(SpatialAudioPlayback) import SpatialAudioPlayback #endif #if canImport(UIKit) @@ -40,7 +40,7 @@ public final class ProAVPlayer { private var fallbackResumesPlayback: Bool? private var playbackTimeObserver: Any? private var externalPlaybackObservation: NSKeyValueObservation? - /// The AtmosOutputActivator while a launch plays APAC on tvOS; releasing it detaches it. + /// The AtmosOutputActivator while a launch plays APAC on tvOS or iOS; releasing it detaches it. private var atmosOutput: AnyObject? private var reportedReady = false private var didFail = false @@ -172,12 +172,12 @@ public final class ProAVPlayer { innerPlayer.prepareToPlay() } - /// On tvOS keeps the receiver in Dolby Atmos while the active launch plays APAC, which APAC alone - /// does not switch. + /// Plays the activator while the active launch plays APAC on this device: APAC alone doesn't switch + /// a receiver to Dolby Atmos, and AirPods render it flatter without it. private func updateSpatialAudioOutput() { - #if os(tvOS) && canImport(SpatialAudioPlayback) - guard #available(tvOS 26, *) else { return } - guard session?.playsSpatialAudio == true else { + #if (os(tvOS) || os(iOS)) && canImport(SpatialAudioPlayback) + guard #available(tvOS 26, iOS 26, *) else { return } + guard session?.playsSpatialAudio == true, !innerPlayer.isExternalPlaybackActive else { atmosOutput = nil return } @@ -217,6 +217,8 @@ public final class ProAVPlayer { private func externalPlaybackDidChange(isActive: Bool) { guard !didFail else { return } + // The activator stops as AirPlay starts, ahead of the restart without APAC. + updateSpatialAudioOutput() guard isActive else { // Back on this device, a launch that left APAC out for AirPlay brings it back. if session?.spatialAudioWithheld == true { diff --git a/docs/03-engine-meplayer-demux-e-pipeline.md b/docs/03-engine-meplayer-demux-e-pipeline.md index 740fd90..5661306 100644 --- a/docs/03-engine-meplayer-demux-e-pipeline.md +++ b/docs/03-engine-meplayer-demux-e-pipeline.md @@ -144,7 +144,7 @@ O `ProAVPlayer` é o terceiro engine do fork. Ele **não decodifica nada**: usa Áudio, por `ProAVAudioStrategy`: EC-3, AC-3, AAC, FLAC e ALAC passam por **stream copy**; qualquer outro codec (DTS, TrueHD, PCM, Opus…) é **transcodificado para FLAC** por `ProAVAudioTranscoder` (S16 ou S32/24 bits, sample rate e layout de canais preservados, PTS regenerado por contagem de amostras). Se o transcoder não puder inicializar, a trilha de áudio é simplesmente descartada. -No tvOS, iOS e macOS 26 ou mais novos, TrueHD com o perfil Atmos do FFmpeg e todo DTS-HD MA a 48 kHz tentam primeiro o **APAC** (o FFmpeg rotula algumas apresentações DTS:X como DTS-HD MA simples): `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Quando a conversão falha, a sessão falha com `spatialAudioFailure` e o `ProAVPlayer` reinicia o launch na posição atual pelo caminho FLAC. Só uma recusa de decodificação da biblioteca (trilha sem Atmos ou DTS:X, ou com metadata que ela não renderiza) fixa o FLAC para aquela trilha da fonte; outras falhas valem só para aquele launch, e o próximo seek tenta o APAC de novo. No tvOS, enquanto o APAC toca, um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. +No tvOS, iOS e macOS 26 ou mais novos, TrueHD com o perfil Atmos do FFmpeg e todo DTS-HD MA a 48 kHz tentam primeiro o **APAC** (o FFmpeg rotula algumas apresentações DTS:X como DTS-HD MA simples): `ProAVSpatialAudioTranscoder` entrega os packets demuxados ao `SpatialAudioTranscoder` do [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge), que renderiza os objetos Atmos ou DTS:X em 7.1.4 e codifica APAC. O vídeo é muxado sozinho, e os fragmentos APAC (`audio-init.mp4`, `audio0.m4s`, …) ganham a própria `audio.m3u8`, anunciada com `URI` no `#EXT-X-MEDIA`, `CHANNELS="12"` e `apac.31.03` no `CODECS`. O muxer rebaseia a timeline para que o primeiro packet de vídeo decodifique em zero; o áudio leva o mesmo deslocamento e começa no primeiro frame de vídeo. A thread de leitura bloqueia enquanto a conversão está atrasada, então o remux anda no ritmo dela, e o gate de prontidão espera também o primeiro fragmento APAC. Quando a conversão falha, a sessão falha com `spatialAudioFailure` e o `ProAVPlayer` reinicia o launch na posição atual pelo caminho FLAC. Só uma recusa de decodificação da biblioteca (trilha sem Atmos ou DTS:X, ou com metadata que ela não renderiza) fixa o FLAC para aquela trilha da fonte; outras falhas valem só para aquele launch, e o próximo seek tenta o APAC de novo. No tvOS e no iOS, enquanto o APAC toca no próprio aparelho (fora do AirPlay), um `AtmosOutputActivator` toca um stream Dolby Digital Plus Atmos silencioso ao lado, que é o que faz o receiver mudar para Atmos. Nos demais casos, a trilha de áudio também é anunciada na master playlist, como uma rendition `#EXT-X-MEDIA:TYPE=AUDIO` sem `URI` (o áudio está muxado na própria variante) referenciada por `AUDIO="main"`. O `CHANNELS` vem de `ch_layout.nb_channels`, **exceto** em E-AC-3 cujo `codecpar.profile` é `AV_PROFILE_EAC3_DDP_ATMOS`: aí o valor é `16/JOC`, que é como a HLS Authoring Specification pede para DD+ com Atmos. Quando a sondagem não fecha a contagem de canais (`nb_channels == 0`), o atributo é **omitido** em vez de sair como `CHANNELS="0"`. From 0a825df831a47e43e3a6ac0d22fb49b7c71c991c Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 19:16:24 +0200 Subject: [PATCH 26/36] docs: note that airpods played the apac as spatial audio --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 9cce48d..e077b5c 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ It ships with a complete SwiftUI interface designed for the Siri Remote, a netwo - Native **Dolby Vision** on the `ProAVPlayer` path — the remux signals `dvh1`/`hvc1` and carries the `dvcC`/`dvvC` configuration box through, so the TV switches into real DV. The FFmpeg engine reports Dolby Vision to the system as HDR10 - **Profile 7 is converted to 8.1** on the way through the remux: the enhancement-layer NAL units are dropped and the RPU is rewritten single-layer with libdovi. Without that conversion the remux engine refuses profile 7 outright and playback falls back to the FFmpeg engine. Implemented and unit-tested; not yet checked against a real Dolby Vision display - E-AC-3, AC-3, AAC, FLAC and ALAC are **bitstream-copied** into the remux, untouched. For Atmos the `moov` is held back until the muxer has parsed an audio frame — which is what lets it write the JOC fields of the `dec3` box — and the playlist declares `CHANNELS="16/JOC"` when the decoder reports the DD+ Atmos profile. Same caveat: the code path is there, the receiver check isn't done -- **TrueHD Atmos and DTS:X play as APAC** on tvOS, iOS and macOS 26 and later: [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and on tvOS and iOS a silent Dolby Digital Plus Atmos stream plays alongside, which switches a receiver to Atmos. A receiver showed Atmos through Lumen on Apple TV; spatial audio on AirPods is still to be tested +- **TrueHD Atmos and DTS:X play as APAC** on tvOS, iOS and macOS 26 and later: [SpatialAudioBridge](https://github.com/kristofferR/SpatialAudioBridge) renders the objects to 7.1.4, the remux serves them as a separate audio rendition, and on tvOS and iOS a silent Dolby Digital Plus Atmos stream plays alongside, which switches a receiver to Atmos. A receiver showed Atmos through Lumen on Apple TV, and AirPods Pro on an iPhone played it as spatial audio with head tracking - Other TrueHD and DTS tracks, and any the library rejects, are re-encoded to FLAC at up to 24-bit: lossless for the channel bed at that depth, but the Atmos objects are gone - MKV, HLS, MP4 and anything else FFmpeg 8.1 demuxes - The `AVPlayer`-backed engines decode in hardware. In the FFmpeg engine, VideoToolbox is opt-in (`asynchronousDecompression`) and falls back to software automatically when a frame fails to decode From 8f0859b6bdfccf5af66bbca5e8d9c28e9e92c70f Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Mon, 12 Oct 2026 21:26:39 +0200 Subject: [PATCH 27/36] build: move to spatialaudiobridge 1.6.1 1.6.1 fixes two DTS:X faults after a seek or a mid-stream start, both in the decoder Lumen's DTS:X path uses: the first frame or two came out distorted (an audible click) and now play as silence, and a dropped pending frame pulled the audio about 11 ms early. --- Package.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Package.swift b/Package.swift index 111068f..84aa368 100644 --- a/Package.swift +++ b/Package.swift @@ -46,5 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), - .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.5.0"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.6.1"), ] From b90e7b876978f601cfff95a81baa39d1b4abb727 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Thu, 15 Oct 2026 02:29:11 +0200 Subject: [PATCH 28/36] fix: preserve audio progress and source switches across airplay --- Package.swift | 2 +- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 14 ++++- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 59 +++++++++++++------ .../Lumen/MEPlayer/ProAVRemuxSession.swift | 26 ++++++-- Tests/LumenTests/ProAVSpatialAudioTest.swift | 59 +++++++++++++++++++ 5 files changed, 135 insertions(+), 25 deletions(-) diff --git a/Package.swift b/Package.swift index 84aa368..e8ad10b 100644 --- a/Package.swift +++ b/Package.swift @@ -46,5 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), - .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.6.1"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.6.2"), ] diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index dc82864..b665f32 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -359,7 +359,7 @@ extension MEPlayerItem { if let audioAssetTrack, let inputAudioStream = formatCtx.pointee.streams[Int(audioAssetTrack.trackID)] { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) let spatialAudioPolicy = session.spatialAudioPolicy - session.spatialAudioWithheld = !spatialAudioPolicy.isEnabled && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) + session.spatialAudioWithheld = spatialAudioPolicy.isEnabled && spatialAudioPolicy.isExternalPlaybackActive && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) if spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { remuxSpatialAudio = spatialAudio audioSignaling = spatialAudio.signaling @@ -445,12 +445,18 @@ extension MEPlayerItem { let index = Int(corePacket.pointee.stream_index) if let remuxSpatialAudio, remuxSpatialAudio.streamIndex == index { remuxSpatialAudio.append(packet: corePacket) + if let inputStream = formatCtx.pointee.streams[index] { + trackRemuxAudioTime(packet: corePacket, stream: inputStream, session: session) + } return } guard let outputIndex = streamMapping[index], let inputStream = formatCtx.pointee.streams[index], let outputStream = outputFormatCtx.pointee.streams[outputIndex] else { return } + if outputStream.pointee.codecpar.pointee.codec_type == AVMEDIA_TYPE_AUDIO { + trackRemuxAudioTime(packet: corePacket, stream: inputStream, session: session) + } if index == remuxTranscodeStreamIndex { guard let remuxTranscoder else { return } let succeeded = remuxTranscoder.transcode(packet: corePacket) { encodedPacket in @@ -520,6 +526,12 @@ extension MEPlayerItem { av_packet_unref(outputPacket) } + private func trackRemuxAudioTime(packet: UnsafeMutablePointer, stream: UnsafeMutablePointer, session: ProAVRemuxSession) { + let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts + guard timestamp != Int64.min else { return } + session.trackAudioTime(seconds: Timebase(stream.pointee.time_base).cmtime(for: timestamp).seconds) + } + private static func requiresParsedHeader(codecID: AVCodecID) -> Bool { codecID == AV_CODEC_ID_AC3 || codecID == AV_CODEC_ID_EAC3 } diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 8649ad6..5a73e0f 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -80,10 +80,10 @@ public final class ProAVPlayer { } } #if !os(xrOS) - externalPlaybackObservation = innerPlayer.player.observe(\.isExternalPlaybackActive) { [weak self] player, _ in - let isActive = player.isExternalPlaybackActive + externalPlaybackObservation = innerPlayer.player.observe(\.isExternalPlaybackActive) { [weak self] _, _ in Task { @MainActor in - self?.externalPlaybackDidChange(isActive: isActive) + guard let self else { return } + self.externalPlaybackDidChange(isActive: self.innerPlayer.isExternalPlaybackActive) } } #endif @@ -110,7 +110,7 @@ public final class ProAVPlayer { session.preferredAudioTrackID = audioTrackID var spatialAudio = spatialAudio // An AirPlay receiver plays the stream itself, so the APAC rendition is left out. - spatialAudio.isEnabled = spatialAudio.isEnabled && !innerPlayer.isExternalPlaybackActive + spatialAudio.isExternalPlaybackActive = innerPlayer.isExternalPlaybackActive session.spatialAudioPolicy = spatialAudio session.playbackTime = time options.startPlayTime = time @@ -145,6 +145,7 @@ public final class ProAVPlayer { private func remuxDidBecomeReady(directoryName: String) { guard !didFail else { return } + guard !restartForExternalPlaybackIfNeeded() else { return } applyDisplayCriteria() if let serverBaseURL { attach(baseURL: serverBaseURL, directoryName: directoryName) @@ -165,7 +166,8 @@ public final class ProAVPlayer { } private func attach(baseURL: URL, directoryName: String) { - guard !didFail else { return } + guard !didFail, directoryName == "launch\(activeLaunchIndex)" else { return } + guard !restartForExternalPlaybackIfNeeded() else { return } let masterURL = baseURL.appendingPathComponent(directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) updateSpatialAudioOutput() innerPlayer.replace(url: masterURL, options: options) @@ -220,24 +222,34 @@ public final class ProAVPlayer { // The activator stops as AirPlay starts, ahead of the restart without APAC. updateSpatialAudioOutput() guard isActive else { - // Back on this device, a launch that left APAC out for AirPlay brings it back. - if session?.spatialAudioWithheld == true { - restartInPlace(spatialAudioEnabled: true) + // Keep the requested source switch. Its readiness callback also rechecks the route if + // opening has not selected audio yet. + if let pending = pendingSourceSwitch { + if pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: false) { + restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) + } + return } + _ = restartForExternalPlaybackIfNeeded() return } if session?.playsSpatialAudio == true { KSLog(level: .info, "AirPlay started; playing the APAC track through FLAC") // Like a seek, the restart abandons a pending switch, which the layer then makes cold, // without APAC while AirPlay lasts. - restartInPlace(spatialAudioEnabled: false) - } else if let pending = pendingSourceSwitch, pending.spatialAudio.isEnabled { - var spatialAudio = pending.spatialAudio - spatialAudio.isEnabled = false - restartPendingSourceSwitch(spatialAudio: spatialAudio) + _ = restartForExternalPlaybackIfNeeded() + } else if let pending = pendingSourceSwitch, pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: true) { + restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) } } + @discardableResult + private func restartForExternalPlaybackIfNeeded() -> Bool { + guard let session, session.needsSpatialAudioRestart(isExternalPlaybackActive: innerPlayer.isExternalPlaybackActive) else { return false } + restartInPlace(spatialAudioEnabled: session.spatialAudioPolicy.isEnabled) + return true + } + /// Restarts the active launch at the current position, keeping the play or pause state. private func restartInPlace(spatialAudioEnabled: Bool) { let seekCompletion = pendingSeekCompletion @@ -274,6 +286,7 @@ public final class ProAVPlayer { fallbackResumesPlayback = nil didFail = false remuxItem?.delegate = nil + atmosOutput = nil innerPlayer.shutdown() remuxItem?.shutdown() session?.requestCleanup() @@ -308,7 +321,7 @@ public final class ProAVPlayer { /// Retries a candidate whose APAC conversion failed on the FLAC path, as a cold launch would. private func pendingSourceSwitchDidFail(_ session: ProAVRemuxSession) { guard let pending = pendingSourceSwitch, let failure = session.spatialAudioFailure else { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) return } var spatialAudio = pending.spatialAudio @@ -333,8 +346,12 @@ public final class ProAVPlayer { private func pendingSourceSwitchDidBecomeReady() { guard let pending = pendingSourceSwitch else { return } + if pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: innerPlayer.isExternalPlaybackActive) { + restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) + return + } guard let serverBaseURL else { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) return } let masterURL = serverBaseURL.appendingPathComponent(pending.launch.directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) @@ -345,7 +362,7 @@ public final class ProAVPlayer { if success { self.commitPendingSourceSwitch() } else { - self.abortPendingSourceSwitch() + self.abortPendingSourceSwitch(restoreSpatialAudio: true) } } } @@ -377,7 +394,7 @@ public final class ProAVPlayer { pending.completion(true) } - private func abortPendingSourceSwitch() { + private func abortPendingSourceSwitch(restoreSpatialAudio: Bool = false) { guard let pending = pendingSourceSwitch else { return } pendingSourceSwitch = nil if pending.innerSwitchStarted { @@ -387,11 +404,15 @@ public final class ProAVPlayer { pending.launch.item.shutdown() pending.launch.session.requestCleanup() pending.completion(false) + // AirPlay may have ended while this candidate was pending, deferring the active restart. + if restoreSpatialAudio, pendingSourceSwitch == nil, !innerPlayer.isExternalPlaybackActive { + _ = restartForExternalPlaybackIfNeeded() + } } fileprivate func launchDidFail(index: Int, error: NSError?) { if let pending = pendingSourceSwitch, pending.launch.index == index { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) return } guard index == activeLaunchIndex else { return } @@ -492,7 +513,7 @@ extension ProAVPlayer: MediaPlayerProtocol { } public func cancelSourceSwitch() { - abortPendingSourceSwitch() + abortPendingSourceSwitch(restoreSpatialAudio: true) } public func shutdown() { diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index aa41315..4117ef3 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -26,9 +26,10 @@ public final class ProAVRemuxSession: @unchecked Sendable { var refusedTrackIDs = Set() /// Off for a launch that retries on FLAC after the conversion failed without refusing the track. var isEnabled = true + var isExternalPlaybackActive = false func allows(trackID: Int32) -> Bool { - isEnabled && !refusedTrackIDs.contains(trackID) + isEnabled && !isExternalPlaybackActive && !refusedTrackIDs.contains(trackID) } } @@ -62,6 +63,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { private var segmentIndex = 0 private var segmentStart: Double? private var lastVideoSeconds: Double? + private var lastAudioSeconds: Double? private var currentSegmentBytes = Int64(0) private var readyFired = false private var finished = false @@ -120,6 +122,13 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + /// Recheck after opening as well as on route changes: the route can change before audio is selected. + func needsSpatialAudioRestart(isExternalPlaybackActive: Bool) -> Bool { + withLock { + isExternalPlaybackActive ? audioSignaling?.mediaPlaylistName != nil : _spatialAudioWithheld + } + } + /// The player's position from the start of the file, which the remux reads ahead of. var playbackTime: TimeInterval? { get { @@ -130,12 +139,15 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } - /// How far the remuxed video runs ahead of playback, once the session is ready and both are known, - /// so the remux always reaches its readiness gate. Video times are the source's own, which start - /// at `sourceStart`. + /// Pace the least advanced required stream so pausing the only reader cannot starve audio. + /// Source timestamps start at `sourceStart`; playback counts from the start of the file. func remuxedAhead(sourceStart: TimeInterval) -> TimeInterval? { withLock { guard readyFired, let lastVideoSeconds, let _playbackTime else { return nil } + if audioSignaling != nil { + guard let lastAudioSeconds else { return nil } + return min(lastVideoSeconds, lastAudioSeconds) - sourceStart - _playbackTime + } return lastVideoSeconds - sourceStart - _playbackTime } } @@ -225,6 +237,12 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } + func trackAudioTime(seconds: Double) { + withLock { + lastAudioSeconds = max(lastAudioSeconds ?? seconds, seconds) + } + } + func closeSegment(nextStartTime: Double) { withLock { guard let start = segmentStart else { diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index 0d6200b..03a8e02 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -192,6 +192,65 @@ class ProAVSpatialAudioTest: XCTestCase { XCTAssertFalse(policy.allows(trackID: 2)) } + func testAirPlayPreservesConversionIntentAndRefusedTracks() { + var policy = ProAVRemuxSession.SpatialAudioPolicy(refusedTrackIDs: [1]) + policy.isExternalPlaybackActive = true + XCTAssertFalse(policy.allows(trackID: 2)) + policy.isExternalPlaybackActive = false + XCTAssertTrue(policy.allows(trackID: 2)) + XCTAssertFalse(policy.allows(trackID: 1)) + policy.isEnabled = false + policy.isExternalPlaybackActive = true + policy.isExternalPlaybackActive = false + XCTAssertFalse(policy.allows(trackID: 2)) + } + + func testOpeningLaunchRechecksAirPlayAfterAudioIsSelected() { + let session = makeSession() + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: true)) + XCTAssertTrue(session.begin(signaling: video, audioSignaling: apac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + XCTAssertTrue(session.needsSpatialAudioRestart(isExternalPlaybackActive: true)) + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: false)) + session.finish(reachedEnd: false) + } + + func testAirPlayWithheldLaunchRetriesOnTheLocalRoute() { + let session = makeSession() + let flac = ProAVAudioSignaling(codecsAttribute: "fLaC", channels: "8") + XCTAssertTrue(session.begin(signaling: video, audioSignaling: flac, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: false)) + session.spatialAudioWithheld = true + XCTAssertTrue(session.needsSpatialAudioRestart(isExternalPlaybackActive: false)) + XCTAssertFalse(session.needsSpatialAudioRestart(isExternalPlaybackActive: true)) + session.finish(reachedEnd: false) + } + + func testReadAheadWaitsForBothVideoAndAudio() { + let flac = ProAVAudioSignaling(codecsAttribute: "fLaC", channels: "8") + for audio in [apac, flac] { + let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: audio, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.playbackTime = 600 + session.trackVideoTime(seconds: 641.4) + for start in [600.0, 602, 604] { + session.closeSegment(nextStartTime: start) + } + if audio.mediaPlaylistName != nil { + session.writeAudioInitSegment(Data([0])) + session.appendAudioSegment(Data([1]), duration: 3.2) + } + XCTAssertNil(session.remuxedAhead(sourceStart: 1.4)) + session.trackAudioTime(seconds: 603.4) + // Forty seconds of video must not pause the reader with only two seconds of audio. + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 2, accuracy: 0.001) + session.trackAudioTime(seconds: 651.4) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 40, accuracy: 0.001) + session.trackAudioTime(seconds: 602) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 40, accuracy: 0.001) + session.finish(reachedEnd: false) + } + } + func testRemuxMeasuresHowFarItReadsAheadOnceReady() { let session = makeSession() XCTAssertTrue(session.begin(signaling: video, audioSignaling: nil, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) From 16eab0e2567f0e20a8b96d9e765ef8253365dd97 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:56:08 +0200 Subject: [PATCH 29/36] fix: preserve the observed airplay route across item replacement --- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 54 ++++++++++++++++++------ 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 5a73e0f..c90a2d2 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -40,6 +40,9 @@ public final class ProAVPlayer { private var fallbackResumesPlayback: Bool? private var playbackTimeObserver: Any? private var externalPlaybackObservation: NSKeyValueObservation? + private var externalPlaybackObservationGeneration = 0 + /// Keep the observed route across item removal, which can clear AVPlayer's active flag. + private var externalPlaybackActive = false /// The AtmosOutputActivator while a launch plays APAC on tvOS or iOS; releasing it detaches it. private var atmosOutput: AnyObject? private var reportedReady = false @@ -79,16 +82,37 @@ public final class ProAVPlayer { self.session?.playbackTime = self.currentPlaybackTime } } + observeExternalPlayback() + } + + private func observeExternalPlayback() { #if !os(xrOS) - externalPlaybackObservation = innerPlayer.player.observe(\.isExternalPlaybackActive) { [weak self] _, _ in + let generation = externalPlaybackObservationGeneration + externalPlaybackObservation = innerPlayer.player.observe(\.isExternalPlaybackActive) { [weak self] player, _ in + let isActive = player.isExternalPlaybackActive Task { @MainActor in - guard let self else { return } - self.externalPlaybackDidChange(isActive: self.innerPlayer.isExternalPlaybackActive) + guard let self, self.externalPlaybackObservationGeneration == generation else { return } + self.externalPlaybackDidChange(isActive: isActive) } } #endif } + /// Replacing an item must not turn a teardown notification into a route change. The generation + /// also drops notifications already queued by the previous observation. + private func updateInnerPlayerItem(_ update: () -> Void) { + externalPlaybackObservation?.invalidate() + externalPlaybackObservation = nil + externalPlaybackObservationGeneration += 1 + update() + // Attachment can activate AirPlay synchronously while observation is suspended. Only an + // observed route change may clear the saved state; item removal's false value cannot. + if innerPlayer.isExternalPlaybackActive { + externalPlaybackActive = true + } + observeExternalPlayback() + } + private static var workspaceRoot: URL { let base = FileManager.default.urls(for: .cachesDirectory, in: .userDomainMask).first ?? FileManager.default.temporaryDirectory return base.appendingPathComponent("Lumen-ProAV") @@ -110,7 +134,7 @@ public final class ProAVPlayer { session.preferredAudioTrackID = audioTrackID var spatialAudio = spatialAudio // An AirPlay receiver plays the stream itself, so the APAC rendition is left out. - spatialAudio.isExternalPlaybackActive = innerPlayer.isExternalPlaybackActive + spatialAudio.isExternalPlaybackActive = externalPlaybackActive session.spatialAudioPolicy = spatialAudio session.playbackTime = time options.startPlayTime = time @@ -170,8 +194,11 @@ public final class ProAVPlayer { guard !restartForExternalPlaybackIfNeeded() else { return } let masterURL = baseURL.appendingPathComponent(directoryName).appendingPathComponent(ProAVRemuxSession.masterPlaylistName) updateSpatialAudioOutput() - innerPlayer.replace(url: masterURL, options: options) - innerPlayer.prepareToPlay() + updateInnerPlayerItem { + innerPlayer.replace(url: masterURL, options: options) + innerPlayer.prepareToPlay() + } + _ = restartForExternalPlaybackIfNeeded() } /// Plays the activator while the active launch plays APAC on this device: APAC alone doesn't switch @@ -179,7 +206,7 @@ public final class ProAVPlayer { private func updateSpatialAudioOutput() { #if (os(tvOS) || os(iOS)) && canImport(SpatialAudioPlayback) guard #available(tvOS 26, iOS 26, *) else { return } - guard session?.playsSpatialAudio == true, !innerPlayer.isExternalPlaybackActive else { + guard session?.playsSpatialAudio == true, !externalPlaybackActive else { atmosOutput = nil return } @@ -218,6 +245,7 @@ public final class ProAVPlayer { } private func externalPlaybackDidChange(isActive: Bool) { + externalPlaybackActive = isActive guard !didFail else { return } // The activator stops as AirPlay starts, ahead of the restart without APAC. updateSpatialAudioOutput() @@ -245,7 +273,7 @@ public final class ProAVPlayer { @discardableResult private func restartForExternalPlaybackIfNeeded() -> Bool { - guard let session, session.needsSpatialAudioRestart(isExternalPlaybackActive: innerPlayer.isExternalPlaybackActive) else { return false } + guard let session, session.needsSpatialAudioRestart(isExternalPlaybackActive: externalPlaybackActive) else { return false } restartInPlace(spatialAudioEnabled: session.spatialAudioPolicy.isEnabled) return true } @@ -269,7 +297,7 @@ public final class ProAVPlayer { seekCompletion?(false) remuxItem?.delegate = nil atmosOutput = nil - innerPlayer.shutdown() + updateInnerPlayerItem { innerPlayer.shutdown() } remuxItem?.shutdown() session?.requestCleanup() server.stop() @@ -287,7 +315,7 @@ public final class ProAVPlayer { didFail = false remuxItem?.delegate = nil atmosOutput = nil - innerPlayer.shutdown() + updateInnerPlayerItem { innerPlayer.shutdown() } remuxItem?.shutdown() session?.requestCleanup() startRemux(at: time, spatialAudioEnabled: spatialAudioEnabled) @@ -346,7 +374,7 @@ public final class ProAVPlayer { private func pendingSourceSwitchDidBecomeReady() { guard let pending = pendingSourceSwitch else { return } - if pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: innerPlayer.isExternalPlaybackActive) { + if pending.launch.session.needsSpatialAudioRestart(isExternalPlaybackActive: externalPlaybackActive) { restartPendingSourceSwitch(spatialAudio: pending.spatialAudio) return } @@ -405,7 +433,7 @@ public final class ProAVPlayer { pending.launch.session.requestCleanup() pending.completion(false) // AirPlay may have ended while this candidate was pending, deferring the active restart. - if restoreSpatialAudio, pendingSourceSwitch == nil, !innerPlayer.isExternalPlaybackActive { + if restoreSpatialAudio, pendingSourceSwitch == nil, !externalPlaybackActive { _ = restartForExternalPlaybackIfNeeded() } } @@ -523,7 +551,7 @@ extension ProAVPlayer: MediaPlayerProtocol { seekCompletion?(false) remuxItem?.delegate = nil atmosOutput = nil - innerPlayer.shutdown() + updateInnerPlayerItem { innerPlayer.shutdown() } remuxItem?.shutdown() session?.requestCleanup() remuxItem = nil From d0688e0e45de4621ed7e9f3b52b0c5dc72dc708f Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:09:08 +0200 Subject: [PATCH 30/36] fix: reset stopped routes and pace video after the audio ends --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 22 ++++++++- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 3 ++ .../Lumen/MEPlayer/ProAVRemuxSession.swift | 12 ++++- Tests/LumenTests/ProAVSpatialAudioTest.swift | 46 +++++++++++++++++++ 4 files changed, 80 insertions(+), 3 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index b665f32..75bca6d 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -26,6 +26,7 @@ public final class MEPlayerItem: Sendable { private var remuxTranscoder: ProAVAudioTranscoder? private var remuxTranscodeStreamIndex: Int? private var remuxSpatialAudio: ProAVSpatialAudioTranscoder? + private var remuxAudioEndTime: TimeInterval? private var remuxCompleted = false private var remuxHeaderWritten = false private var remuxMoovWritten = false @@ -388,6 +389,7 @@ extension MEPlayerItem { KSLog("ProAV audio track skipped: flac transcode unavailable") } if streamMapping[Int(audioAssetTrack.trackID)] != nil || remuxSpatialAudio != nil { + remuxAudioEndTime = Self.audioEndTime(stream: inputAudioStream.pointee, sourceStart: startTime.seconds) audios.forEach { $0.isEnabled = $0.trackID == audioAssetTrack.trackID } } } @@ -529,7 +531,25 @@ extension MEPlayerItem { private func trackRemuxAudioTime(packet: UnsafeMutablePointer, stream: UnsafeMutablePointer, session: ProAVRemuxSession) { let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts guard timestamp != Int64.min else { return } - session.trackAudioTime(seconds: Timebase(stream.pointee.time_base).cmtime(for: timestamp).seconds) + let timebase = Timebase(stream.pointee.time_base) + session.trackAudioTime(seconds: timebase.cmtime(for: timestamp).seconds, duration: timebase.cmtime(for: max(packet.pointee.duration, 0)).seconds, streamEnd: remuxAudioEndTime) + } + + /// Matroska's DURATION tag is the track's final timestamp, including the last packet's duration. + static func audioEndTime(stream: AVStream, sourceStart: TimeInterval) -> TimeInterval? { + if let tag = toDictionary(stream.metadata)["DURATION"] { + let parts = tag.split(separator: ":", omittingEmptySubsequences: false) + if parts.count == 3, let hours = Double(parts[0]), let minutes = Double(parts[1]), let seconds = Double(parts[2]), + hours >= 0, minutes >= 0, minutes < 60, seconds >= 0, seconds < 60 + { + let end = hours * 3600 + minutes * 60 + seconds + if end.isFinite, end > 0 { return end } + } + } + guard stream.duration > 0, stream.time_base.num > 0, stream.time_base.den > 0 else { return nil } + let timebase = Timebase(stream.time_base) + let start = stream.start_time == Int64.min ? sourceStart : timebase.cmtime(for: stream.start_time).seconds + return start + timebase.cmtime(for: stream.duration).seconds } private static func requiresParsedHeader(codecID: AVCodecID) -> Bool { diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index c90a2d2..9ea5610 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -503,6 +503,8 @@ extension ProAVPlayer: MediaPlayerProtocol { public var pipController: KSPictureInPictureController? { innerPlayer.pipController } public func prepareToPlay() { + // A fresh play must not inherit the route from a previous, terminal shutdown. + externalPlaybackActive = innerPlayer.isExternalPlaybackActive didFail = false startRemux(at: options.startPlayTime) } @@ -558,6 +560,7 @@ extension ProAVPlayer: MediaPlayerProtocol { session = nil server.stop() serverBaseURL = nil + externalPlaybackActive = false } public func seek(time: TimeInterval, completion: @escaping ((Bool) -> Void)) { diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 4117ef3..799c904 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -64,6 +64,7 @@ public final class ProAVRemuxSession: @unchecked Sendable { private var segmentStart: Double? private var lastVideoSeconds: Double? private var lastAudioSeconds: Double? + private var audioEndSeconds: Double? private var currentSegmentBytes = Int64(0) private var readyFired = false private var finished = false @@ -146,6 +147,11 @@ public final class ProAVRemuxSession: @unchecked Sendable { guard readyFired, let lastVideoSeconds, let _playbackTime else { return nil } if audioSignaling != nil { guard let lastAudioSeconds else { return nil } + // Once all declared audio is read, its shorter duration must not make the video + // tail read without a bound. Unknown ends still wait for audio to avoid starvation. + if let audioEndSeconds, lastAudioSeconds >= audioEndSeconds - 0.000001 { + return lastVideoSeconds - sourceStart - _playbackTime + } return min(lastVideoSeconds, lastAudioSeconds) - sourceStart - _playbackTime } return lastVideoSeconds - sourceStart - _playbackTime @@ -237,9 +243,11 @@ public final class ProAVRemuxSession: @unchecked Sendable { } } - func trackAudioTime(seconds: Double) { + func trackAudioTime(seconds: Double, duration: Double = 0, streamEnd: Double? = nil) { withLock { - lastAudioSeconds = max(lastAudioSeconds ?? seconds, seconds) + let end = seconds + max(duration, 0) + lastAudioSeconds = max(lastAudioSeconds ?? end, end) + audioEndSeconds = streamEnd } } diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index 03a8e02..2330ac1 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -251,6 +251,52 @@ class ProAVSpatialAudioTest: XCTestCase { } } + func testReadAheadUsesVideoAfterTheDeclaredAudioEnd() { + let flac = ProAVAudioSignaling(codecsAttribute: "fLaC", channels: "8") + for audio in [apac, flac] { + let session = makeSession() + XCTAssertTrue(session.begin(signaling: video, audioSignaling: audio, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) + session.playbackTime = 600 + session.trackVideoTime(seconds: 641.4) + for start in [600.0, 602, 604] { + session.closeSegment(nextStartTime: start) + } + if audio.mediaPlaylistName != nil { + session.writeAudioInitSegment(Data([0])) + session.appendAudioSegment(Data([1]), duration: 3.2) + } + session.trackAudioTime(seconds: 603.4, duration: 1, streamEnd: 605.4) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 3, accuracy: 0.001) + // The last packet reaches the track's declared end, so a long video-only tail can pause. + session.trackAudioTime(seconds: 604.4, duration: 1, streamEnd: 605.4) + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 40, accuracy: 0.001) + session.playbackTime = 620 + XCTAssertEqual(session.remuxedAhead(sourceStart: 1.4) ?? -1, 20, accuracy: 0.001) + session.finish(reachedEnd: false) + } + } + + func testAudioEndUsesTrackDurationOrMatroskaEndTag() throws { + let formatContext = avformat_alloc_context() + defer { avformat_free_context(formatContext) } + let stream = try XCTUnwrap(avformat_new_stream(formatContext, nil)) + stream.pointee.time_base = AVRational(num: 1, den: 1000) + stream.pointee.start_time = 1000 + stream.pointee.duration = 5000 + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 6) + stream.pointee.start_time = Int64.min + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 15) + XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "00:00:03.250000000", 0), 0) + // Matroska stores an absolute end timestamp, not a duration to add to start_time. + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 3.25) + XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "unknown", 0), 0) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 15) + XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "bad:00:00:03.250", 0), 0) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 15) + stream.pointee.duration = Int64.min + XCTAssertNil(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10)) + } + func testRemuxMeasuresHowFarItReadsAheadOnceReady() { let session = makeSession() XCTAssertTrue(session.begin(signaling: video, audioSignaling: nil, bandwidth: 12_000_000, resolution: CGSize(width: 1920, height: 1080), frameRate: 23.976)) From cb7d9eb3365c26e740d0961448a8fee3ee41d69d Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:18:14 +0200 Subject: [PATCH 31/36] fix: retain queued route changes and scope matroska end tags --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 8 +++++--- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 5 +++++ Tests/LumenTests/ProAVSpatialAudioTest.swift | 16 ++++++++++------ 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 75bca6d..c886e92 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -389,7 +389,7 @@ extension MEPlayerItem { KSLog("ProAV audio track skipped: flac transcode unavailable") } if streamMapping[Int(audioAssetTrack.trackID)] != nil || remuxSpatialAudio != nil { - remuxAudioEndTime = Self.audioEndTime(stream: inputAudioStream.pointee, sourceStart: startTime.seconds) + remuxAudioEndTime = Self.audioEndTime(stream: inputAudioStream.pointee, sourceStart: startTime.seconds, formatName: options.formatName) audios.forEach { $0.isEnabled = $0.trackID == audioAssetTrack.trackID } } } @@ -536,8 +536,10 @@ extension MEPlayerItem { } /// Matroska's DURATION tag is the track's final timestamp, including the last packet's duration. - static func audioEndTime(stream: AVStream, sourceStart: TimeInterval) -> TimeInterval? { - if let tag = toDictionary(stream.metadata)["DURATION"] { + static func audioEndTime(stream: AVStream, sourceStart: TimeInterval, formatName: String) -> TimeInterval? { + if formatName.split(separator: ",").contains(where: { $0 == "matroska" || $0 == "webm" }), + let tag = toDictionary(stream.metadata)["DURATION"] + { let parts = tag.split(separator: ":", omittingEmptySubsequences: false) if parts.count == 3, let hours = Double(parts[0]), let minutes = Double(parts[1]), let seconds = Double(parts[2]), hours >= 0, minutes >= 0, minutes < 60, seconds >= 0, seconds < 60 diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index 9ea5610..a3e8262 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -101,6 +101,11 @@ public final class ProAVPlayer { /// Replacing an item must not turn a teardown notification into a route change. The generation /// also drops notifications already queued by the previous observation. private func updateInnerPlayerItem(_ update: () -> Void) { + // Capture a real disconnect that may still be queued on the main actor. Once the item is + // gone, a false property value can instead be a side effect of our own teardown. + if innerPlayer.player.currentItem != nil { + externalPlaybackActive = innerPlayer.isExternalPlaybackActive + } externalPlaybackObservation?.invalidate() externalPlaybackObservation = nil externalPlaybackObservationGeneration += 1 diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index 2330ac1..d0b6fd9 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -283,18 +283,22 @@ class ProAVSpatialAudioTest: XCTestCase { stream.pointee.time_base = AVRational(num: 1, den: 1000) stream.pointee.start_time = 1000 stream.pointee.duration = 5000 - XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 6) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 6) stream.pointee.start_time = Int64.min - XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 15) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 15) XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "00:00:03.250000000", 0), 0) // Matroska stores an absolute end timestamp, not a duration to add to start_time. - XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 3.25) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 3.25) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "mov,mp4,m4a,3gp,3g2,mj2"), 15) + stream.pointee.duration = Int64.min + XCTAssertNil(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "mpegts")) + stream.pointee.duration = 5000 XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "unknown", 0), 0) - XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 15) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 15) XCTAssertEqual(av_dict_set(&stream.pointee.metadata, "DURATION", "bad:00:00:03.250", 0), 0) - XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10), 15) + XCTAssertEqual(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm"), 15) stream.pointee.duration = Int64.min - XCTAssertNil(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10)) + XCTAssertNil(MEPlayerItem.audioEndTime(stream: stream.pointee, sourceStart: 10, formatName: "matroska,webm")) } func testRemuxMeasuresHowFarItReadsAheadOnceReady() { From f5f0f6121666ff9571f527d29f10d8ccc8dd68cb Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Thu, 15 Oct 2026 13:16:36 +0200 Subject: [PATCH 32/36] fix: avoid creating the playback timer during layer teardown --- Sources/Lumen/AVPlayer/KSPlayerLayer.swift | 31 +++++++++++++--------- Tests/LumenTests/SourceSwitchTest.swift | 14 ++++++++++ 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/Sources/Lumen/AVPlayer/KSPlayerLayer.swift b/Sources/Lumen/AVPlayer/KSPlayerLayer.swift index 359d8b5..d6411a8 100644 --- a/Sources/Lumen/AVPlayer/KSPlayerLayer.swift +++ b/Sources/Lumen/AVPlayer/KSPlayerLayer.swift @@ -171,18 +171,24 @@ open class KSPlayerLayer: NSObject { } } - private lazy var timer: Timer = .scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in - guard let self, self.player.isReadyToPlay else { - return - } - self.delegate?.player(layer: self, currentTime: self.player.currentPlaybackTime, totalTime: self.player.duration) - if self.player.playbackState == .playing, self.player.loadState == .playable, self.state == .buffering { - // 一个兜底保护，正常不能走到这里 - self.state = .bufferFinished - } - if self.player.isPlaying { - MPNowPlayingInfoCenter.default().nowPlayingInfo?[MPNowPlayingInfoPropertyElapsedPlaybackTime] = self.player.currentPlaybackTime + private var playbackTimer: Timer? + private var timer: Timer { + if let playbackTimer { return playbackTimer } + let timer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in + guard let self, self.player.isReadyToPlay else { + return + } + self.delegate?.player(layer: self, currentTime: self.player.currentPlaybackTime, totalTime: self.player.duration) + if self.player.playbackState == .playing, self.player.loadState == .playable, self.state == .buffering { + // 一个兜底保护，正常不能走到这里 + self.state = .bufferFinished + } + if self.player.isPlaying { + MPNowPlayingInfoCenter.default().nowPlayingInfo?[MPNowPlayingInfoPropertyElapsedPlaybackTime] = self.player.currentPlaybackTime + } } + playbackTimer = timer + return timer } private var urls = [URL]() @@ -239,7 +245,8 @@ open class KSPlayerLayer: NSObject { } deinit { - timer.invalidate() + // Cleanup must not initialize a timer whose closure captures a deallocating self. + playbackTimer?.invalidate() if #available(iOS 15.0, tvOS 15.0, macOS 12.0, *) { player.pipController?.contentSource = nil } diff --git a/Tests/LumenTests/SourceSwitchTest.swift b/Tests/LumenTests/SourceSwitchTest.swift index 4131d6a..9f7cfc0 100644 --- a/Tests/LumenTests/SourceSwitchTest.swift +++ b/Tests/LumenTests/SourceSwitchTest.swift @@ -151,6 +151,20 @@ class SourceSwitchTest: XCTestCase { wait(for: [drained], timeout: 2) } + @MainActor + func testLayerCanBeReleasedBeforePlaybackStarts() throws { + KSOptions.firstPlayerType = ColdOnlyFakeEngine.self + let url = try XCTUnwrap(URL(string: "https://example.com/a.m3u8")) + let options = KSOptions() + options.registerRemoteControll = false + weak var releasedLayer: KSPlayerLayer? + do { + let layer = KSPlayerLayer(url: url, isAutoPlay: false, options: options) + releasedLayer = layer + } + XCTAssertNil(releasedLayer) + } + @MainActor func testDefaultSwitchSourceFallsBackToColdPath() throws { KSOptions.firstPlayerType = ColdOnlyFakeEngine.self From 8df92baa9bd0382b0210bff3982ad5698d645810 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:26:51 +0200 Subject: [PATCH 33/36] build: move to spatialaudiobridge 2.0.0 2.0 reports tracks it will never render as unsupported rather than as a decoding error, so refusal memory now matches that class. Every other error, including a discontinuity, falls back for the launch only: the point lies ahead of playback, so the next seek starts a new session. Sync points come from the library's own check. --- Package.swift | 2 +- .../Lumen/MEPlayer/ProAVRemuxSession.swift | 4 +- .../ProAVSpatialAudioTranscoder.swift | 33 ++++++---------- Tests/LumenTests/ProAVSpatialAudioTest.swift | 39 +++++++------------ 4 files changed, 30 insertions(+), 48 deletions(-) diff --git a/Package.swift b/Package.swift index e8ad10b..f6da5c0 100644 --- a/Package.swift +++ b/Package.swift @@ -46,5 +46,5 @@ let package = Package( package.dependencies += [ .package(path: "FFmpegKit"), - .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "1.6.2"), + .package(url: "https://github.com/kristofferR/SpatialAudioBridge", from: "2.0.0"), ] diff --git a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift index 799c904..1b3ac1a 100644 --- a/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift +++ b/Sources/Lumen/MEPlayer/ProAVRemuxSession.swift @@ -304,10 +304,10 @@ public final class ProAVRemuxSession: @unchecked Sendable { withLock { failLocked(error) } } - func failSpatialAudio(_ error: NSError, refusedTrackID: Int32?) { + func failSpatialAudio(_ error: NSError, failure: SpatialAudioFailure) { withLock { guard !failed else { return } - _spatialAudioFailure = refusedTrackID.map { .refused(trackID: $0) } ?? .failed + _spatialAudioFailure = failure failLocked(error) } } diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift index 01b4a70..fdbc797 100644 --- a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -120,7 +120,7 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { return nil } // FFmpeg reports some DTS:X presentations as plain DTS-HD MA, so every one is offered and the - // library tells them apart once it decodes. TrueHD goes by FFmpeg's Atmos label. + // library tells them apart by decoding. TrueHD goes by FFmpeg's Atmos label. switch codecpar.codec_id { case AV_CODEC_ID_TRUEHD where codecpar.profile == AV_PROFILE_TRUEHD_ATMOS: return .trueHDAtmos @@ -138,12 +138,12 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { var held = [(data: Data, time: CMTime?)]() var unstartedBytes = 0 var failed = false - // Any error ends the session at once, so the player restarts on the FLAC path; a decoding - // refusal keeps the track there. + // Any error ends the session at once, so the player restarts on the FLAC path; an unsupported + // track stays there. func fail(_ error: Error) { failed = true guard !isCancelled else { return } - session.failSpatialAudio(NSError(description: "ProAV spatial audio failed: \(error)"), refusedTrackID: Self.refusesTrack(error) ? Int32(streamIndex) : nil) + session.failSpatialAudio(NSError(description: "ProAV spatial audio failed: \(error)"), failure: Self.failure(for: error, trackID: Int32(streamIndex))) } func append(_ data: Data, time: CMTime?) async throws { guard let timeline else { @@ -153,7 +153,7 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { if transcoder == nil { // Decoding starts at a TrueHD major sync or a DTS core sync; earlier audio is discarded // and a later start is padded with silence, so the audio begins with the video. - guard let time, Self.isSyncPoint(data, codec: codec) else { return } + guard let time, codec.isSyncPoint(data) else { return } var settings = configuration settings.startTime = timeline.start settings.inputStartTime = time - timeline.origin @@ -215,24 +215,15 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { completion.signal() } - /// Whether the library refused the track itself, rather than failing to convert it this time. + /// How the player responds: an unsupported track keeps FLAC for its source, and anything else + /// falls back for this launch only. That includes a discontinuity: its point lies ahead of + /// playback, so restarting the launch would meet it again, and the next seek starts a new session. @available(macOS 26, iOS 26, tvOS 26, *) - static func refusesTrack(_ error: Error) -> Bool { - if case .decoding? = error as? SpatialAudioError { - return true - } - return false - } - - @available(macOS 26, iOS 26, tvOS 26, *) - static func isSyncPoint(_ data: Data, codec: SpatialAudioCodec) -> Bool { - let bytes = [UInt8](data.prefix(8)) - switch codec { - case .trueHDAtmos: - return bytes.count == 8 && bytes[4 ... 7] == [0xF8, 0x72, 0x6F, 0xBA] - case .dtsX: - return bytes.starts(with: [0x7F, 0xFE, 0x80, 0x01]) + static func failure(for error: Error, trackID: Int32) -> ProAVRemuxSession.SpatialAudioFailure { + if case .unsupported? = error as? SpatialAudioError { + return .refused(trackID: trackID) } + return .failed } #endif } diff --git a/Tests/LumenTests/ProAVSpatialAudioTest.swift b/Tests/LumenTests/ProAVSpatialAudioTest.swift index d0b6fd9..2bae8da 100644 --- a/Tests/LumenTests/ProAVSpatialAudioTest.swift +++ b/Tests/LumenTests/ProAVSpatialAudioTest.swift @@ -53,33 +53,24 @@ class ProAVSpatialAudioTest: XCTestCase { #endif } - func testConversionStartsAtASyncPoint() throws { + func testOnlyUnsupportedTracksAreRefused() throws { #if canImport(SpatialAudioBridge) guard #available(macOS 26, iOS 26, tvOS 26, *) else { throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") } - let majorSync = Data([0x12, 0x34, 0x56, 0x78, 0xF8, 0x72, 0x6F, 0xBA, 0x00]) - let mlpMajorSync = Data([0x12, 0x34, 0x56, 0x78, 0xF8, 0x72, 0x6F, 0xBB, 0x00]) - XCTAssertTrue(ProAVSpatialAudioTranscoder.isSyncPoint(majorSync, codec: .trueHDAtmos)) - XCTAssertFalse(ProAVSpatialAudioTranscoder.isSyncPoint(mlpMajorSync, codec: .trueHDAtmos)) - XCTAssertFalse(ProAVSpatialAudioTranscoder.isSyncPoint(majorSync.prefix(6), codec: .trueHDAtmos)) - XCTAssertTrue(ProAVSpatialAudioTranscoder.isSyncPoint(Data([0x7F, 0xFE, 0x80, 0x01, 0xFC]), codec: .dtsX)) - XCTAssertFalse(ProAVSpatialAudioTranscoder.isSyncPoint(Data([0xFE, 0x7F, 0x01, 0x80, 0xFC]), codec: .dtsX)) - #else - throw XCTSkip("SpatialAudioBridge is unavailable on this platform") - #endif - } - - func testOnlyDecodingErrorsRefuseTheTrack() throws { - #if canImport(SpatialAudioBridge) - guard #available(macOS 26, iOS 26, tvOS 26, *) else { - throw XCTSkip("SpatialAudioBridge requires macOS, iOS or tvOS 26") + XCTAssertEqual(ProAVSpatialAudioTranscoder.failure(for: SpatialAudioError.unsupported("This TrueHD stream has no Atmos presentation"), trackID: 1), .refused(trackID: 1)) + // A discontinuity lies ahead of playback, so like a transient error it falls back for the launch. + let launchOnly: [Error] = [ + SpatialAudioError.discontinuity("Timestamps jumped"), + SpatialAudioError.decoding("Atmos presentation has no object metadata within 0.5 s"), + SpatialAudioError.encoding("APAC encoder stopped"), + SpatialAudioError.source("No sync point or video timeline within 8 MB of audio"), + SpatialAudioError.coreMedia(-12780), + CancellationError(), + ] + for error in launchOnly { + XCTAssertEqual(ProAVSpatialAudioTranscoder.failure(for: error, trackID: 1), .failed, "\(error)") } - XCTAssertTrue(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.decoding("This TrueHD stream has no Atmos presentation"))) - XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.encoding("APAC encoder stopped"))) - XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.source("No sync point or video timeline within 8 MB of audio"))) - XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(SpatialAudioError.coreMedia(-12780))) - XCTAssertFalse(ProAVSpatialAudioTranscoder.refusesTrack(CancellationError())) #else throw XCTSkip("SpatialAudioBridge is unavailable on this platform") #endif @@ -175,11 +166,11 @@ class ProAVSpatialAudioTest: XCTestCase { func testOnlySpatialAudioFailuresFallBackToFLAC() { let failed = makeSession() failed.fail(NSError(description: "segment write failed")) - failed.failSpatialAudio(NSError(description: "unsupported metadata"), refusedTrackID: 1) + failed.failSpatialAudio(NSError(description: "unsupported metadata"), failure: .refused(trackID: 1)) XCTAssertTrue(failed.isFailed) XCTAssertNil(failed.spatialAudioFailure) let refused = makeSession() - refused.failSpatialAudio(NSError(description: "unsupported metadata"), refusedTrackID: 1) + refused.failSpatialAudio(NSError(description: "unsupported metadata"), failure: .refused(trackID: 1)) XCTAssertTrue(refused.isFailed) XCTAssertEqual(refused.spatialAudioFailure, .refused(trackID: 1)) } From 3661ed088bb154afc1c35907bf34ba618ef77691 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:26:51 +0200 Subject: [PATCH 34/36] feat: probe spatial audio tracks before the launch A track the library refuses, such as DTS-HD MA without DTS:X, used to start the APAC conversion and fail the launch, which then reopened on FLAC. The launch now decodes about a second of the track first and keeps a refused one on FLAC from the start. --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 25 +++++++++- .../ProAVSpatialAudioTranscoder.swift | 50 ++++++++++++++++++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index c886e92..a1bae5d 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -361,7 +361,7 @@ extension MEPlayerItem { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) let spatialAudioPolicy = session.spatialAudioPolicy session.spatialAudioWithheld = spatialAudioPolicy.isEnabled && spatialAudioPolicy.isExternalPlaybackActive && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) - if spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { + if spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), !probeRefusesSpatialAudio(stream: inputAudioStream, formatCtx: formatCtx), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { remuxSpatialAudio = spatialAudio audioSignaling = spatialAudio.signaling } else if strategy.copiesBitstream { @@ -420,6 +420,29 @@ extension MEPlayerItem { outputPacket = av_packet_alloc() } + /// Whether the library will refuse a track the conversion would take, such as DTS-HD MA without + /// DTS:X, so it plays through FLAC from the start instead of failing the launch. The probe reads + /// ahead, so it needs a seekable source, and a launch from the start rewinds afterwards; the read + /// thread seeks a later start itself. + private func probeRefusesSpatialAudio(stream: UnsafeMutablePointer, formatCtx: UnsafeMutablePointer) -> Bool { + guard seekable, let codecpar = stream.pointee.codecpar, ProAVSpatialAudioTranscoder.converts(codecpar: codecpar.pointee) else { + return false + } + let refusal = ProAVSpatialAudioTranscoder.probeRefusal(stream: stream, formatContext: formatCtx) + if options.startPlayTime <= 0 { + let start = seekByBytes || formatCtx.pointee.start_time == Int64.min ? 0 : formatCtx.pointee.start_time + let result = avformat_seek_file(formatCtx, -1, Int64.min, start, Int64.max, seekByBytes ? AVSEEK_FLAG_BYTE : 0) + if result < 0 { + KSLog("ProAV spatial audio probe could not rewind: \(result)") + } + } + guard let refusal else { + return false + } + KSLog(level: .info, "ProAV spatial audio unsupported: \(refusal); playing FLAC") + return true + } + private func overrideDolbyVisionConfigurationRecord(codecpar: UnsafeMutablePointer) -> Bool { let recordSize = MemoryLayout.size var source = [UInt8]() diff --git a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift index fdbc797..4758949 100644 --- a/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift +++ b/Sources/Lumen/MEPlayer/ProAVSpatialAudioTranscoder.swift @@ -7,8 +7,9 @@ import SpatialAudioBridge #endif /// Converts a TrueHD Atmos or DTS:X track to APAC, which the remux serves as a separate audio -/// rendition instead of the FLAC transcode. A track without Atmos or DTS:X fails the session as -/// decoding starts, and the player then plays it through the FLAC transcode. +/// rendition instead of the FLAC transcode. A probe before the launch keeps a track the library refuses +/// on FLAC; one the probe cannot judge fails the session as decoding starts, and the player then +/// plays it through the FLAC transcode. /// /// The read thread hands packets over and blocks while the conversion is behind, so the remux runs at /// its pace. Fragments are placed on the muxer's timeline once `begin(origin:videoStart:)` says where @@ -113,6 +114,51 @@ final class ProAVSpatialAudioTranscoder: @unchecked Sendable { return false } + /// Why the library will never render the track, found by decoding about a second of it from its + /// first sync point without encoding; nil when it renders or the probe cannot tell. Reads the + /// demuxer onward from where it is. + static func probeRefusal(stream: UnsafeMutablePointer, formatContext: UnsafeMutablePointer) -> String? { + #if canImport(SpatialAudioBridge) + guard #available(macOS 26, iOS 26, tvOS 26, *), + let codecpar = stream.pointee.codecpar, + let codec = codec(codecpar: codecpar.pointee) + else { + return nil + } + var packet = av_packet_alloc() + defer { av_packet_free(&packet) } + guard let packet else { return nil } + let timebase = Timebase(stream.pointee.time_base) + var audio = Data() + var start: CMTime? + var bytesRead = 0 + // The video read around the audio counts as well, so a track without a sync point ends it. + while bytesRead < 32 << 20, audio.count < 8 << 20, av_read_frame(formatContext, packet) == 0 { + defer { av_packet_unref(packet) } + bytesRead += Int(packet.pointee.size) + guard packet.pointee.stream_index == stream.pointee.index, let bytes = packet.pointee.data, packet.pointee.size > 0 else { continue } + let data = Data(bytes: bytes, count: Int(packet.pointee.size)) + guard !audio.isEmpty || codec.isSyncPoint(data) else { continue } + let timestamp = packet.pointee.pts == Int64.min ? packet.pointee.dts : packet.pointee.pts + if timestamp != Int64.min { + let time = timebase.cmtime(for: timestamp) + if let start, (time - start).seconds >= 1 { + break + } + start = start ?? time + } + audio += data + } + // Damaged audio is for the conversion to judge, which falls back for this launch only. + guard case let .unsupported(reason)? = try? codec.probe(audio) else { + return nil + } + return reason + #else + return nil + #endif + } + #if canImport(SpatialAudioBridge) @available(macOS 26, iOS 26, tvOS 26, *) static func codec(codecpar: AVCodecParameters) -> SpatialAudioCodec? { From c3b504ce0d0d1ebbf572df76028571ef838d7f66 Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:32:29 +0200 Subject: [PATCH 35/36] fix: fail the launch when the spatial audio probe cannot rewind --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index a1bae5d..8c0fb8c 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -361,7 +361,9 @@ extension MEPlayerItem { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) let spatialAudioPolicy = session.spatialAudioPolicy session.spatialAudioWithheld = spatialAudioPolicy.isEnabled && spatialAudioPolicy.isExternalPlaybackActive && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) - if spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID), !probeRefusesSpatialAudio(stream: inputAudioStream, formatCtx: formatCtx), let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { + let triesSpatialAudio = spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID) && !probeRefusesSpatialAudio(stream: inputAudioStream, formatCtx: formatCtx) + guard error == nil else { return } + if triesSpatialAudio, let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { remuxSpatialAudio = spatialAudio audioSignaling = spatialAudio.signaling } else if strategy.copiesBitstream { @@ -423,7 +425,7 @@ extension MEPlayerItem { /// Whether the library will refuse a track the conversion would take, such as DTS-HD MA without /// DTS:X, so it plays through FLAC from the start instead of failing the launch. The probe reads /// ahead, so it needs a seekable source, and a launch from the start rewinds afterwards; the read - /// thread seeks a later start itself. + /// thread seeks a later start itself. A launch that cannot rewind fails rather than skip the start. private func probeRefusesSpatialAudio(stream: UnsafeMutablePointer, formatCtx: UnsafeMutablePointer) -> Bool { guard seekable, let codecpar = stream.pointee.codecpar, ProAVSpatialAudioTranscoder.converts(codecpar: codecpar.pointee) else { return false @@ -433,7 +435,8 @@ extension MEPlayerItem { let start = seekByBytes || formatCtx.pointee.start_time == Int64.min ? 0 : formatCtx.pointee.start_time let result = avformat_seek_file(formatCtx, -1, Int64.min, start, Int64.max, seekByBytes ? AVSEEK_FLAG_BYTE : 0) if result < 0 { - KSLog("ProAV spatial audio probe could not rewind: \(result)") + error = NSError(description: "ProAV spatial audio probe could not rewind: \(result)") + return false } } guard let refusal else { From c62594a4ac5756bd7aafbe9268ecb5191ad0c72d Mon Sep 17 00:00:00 2001 From: kristofferR <481270+kristofferR@users.noreply.github.com> Date: Sat, 17 Oct 2026 14:37:42 +0200 Subject: [PATCH 36/36] fix: remember tracks the spatial audio probe refused --- Sources/Lumen/MEPlayer/MEPlayerItem.swift | 7 ++++++- Sources/Lumen/MEPlayer/ProAVPlayer.swift | 8 ++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/Sources/Lumen/MEPlayer/MEPlayerItem.swift b/Sources/Lumen/MEPlayer/MEPlayerItem.swift index 8c0fb8c..63aa6c3 100644 --- a/Sources/Lumen/MEPlayer/MEPlayerItem.swift +++ b/Sources/Lumen/MEPlayer/MEPlayerItem.swift @@ -361,7 +361,12 @@ extension MEPlayerItem { let strategy = ProAVAudioStrategy.make(codecpar: audioAssetTrack.codecpar) let spatialAudioPolicy = session.spatialAudioPolicy session.spatialAudioWithheld = spatialAudioPolicy.isEnabled && spatialAudioPolicy.isExternalPlaybackActive && !spatialAudioPolicy.refusedTrackIDs.contains(audioAssetTrack.trackID) && ProAVSpatialAudioTranscoder.converts(codecpar: audioAssetTrack.codecpar) - let triesSpatialAudio = spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID) && !probeRefusesSpatialAudio(stream: inputAudioStream, formatCtx: formatCtx) + var triesSpatialAudio = spatialAudioPolicy.allows(trackID: audioAssetTrack.trackID) + if triesSpatialAudio, probeRefusesSpatialAudio(stream: inputAudioStream, formatCtx: formatCtx) { + triesSpatialAudio = false + // Later launches of this source skip the probe. + session.spatialAudioPolicy.refusedTrackIDs.insert(audioAssetTrack.trackID) + } guard error == nil else { return } if triesSpatialAudio, let spatialAudio = ProAVSpatialAudioTranscoder(stream: inputAudioStream, session: session) { remuxSpatialAudio = spatialAudio diff --git a/Sources/Lumen/MEPlayer/ProAVPlayer.swift b/Sources/Lumen/MEPlayer/ProAVPlayer.swift index a3e8262..722fc64 100644 --- a/Sources/Lumen/MEPlayer/ProAVPlayer.swift +++ b/Sources/Lumen/MEPlayer/ProAVPlayer.swift @@ -33,7 +33,7 @@ public final class ProAVPlayer { private var delegateProxy: ProAVLaunchDelegateProxy? private var pendingSourceSwitch: PendingSourceSwitch? private var preferredAudioTrackID: Int32? - /// Audio tracks of this source that the library refused; they keep the FLAC path. + /// Audio tracks of this source that the library or a launch's probe refused; they keep the FLAC path. private var refusedSpatialAudioTrackIDs = Set() /// Whether playback resumes after an in-place restart, for the FLAC fallback or AirPlay, of a launch /// that was already playing. @@ -312,6 +312,10 @@ public final class ProAVPlayer { private func restart(at time: TimeInterval, completion: ((Bool) -> Void)?, spatialAudioEnabled: Bool = true) { abortPendingSourceSwitch() + // Keep the tracks the launch's probe refused. + if let session { + refusedSpatialAudioTrackIDs.formUnion(session.spatialAudioPolicy.refusedTrackIDs) + } let previousCompletion = pendingSeekCompletion pendingSeekCompletion = nil previousCompletion?(false) @@ -411,7 +415,7 @@ public final class ProAVPlayer { url = pending.url options = pending.options preferredAudioTrackID = nil - refusedSpatialAudioTrackIDs = pending.spatialAudio.refusedTrackIDs + refusedSpatialAudioTrackIDs = pending.launch.session.spatialAudioPolicy.refusedTrackIDs session = pending.launch.session remuxItem = pending.launch.item delegateProxy = pending.launch.proxy