Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
94 changes: 64 additions & 30 deletions ios/EternalMonitor/App/ConnectionManager.swift
Original file line number Diff line number Diff line change
Expand Up @@ -408,32 +408,39 @@ final class ConnectionManager: ObservableObject {
try? await Task.sleep(nanoseconds: 250_000_000)
guard let self, !Task.isCancelled else { return }
let snapshot = self.streamCounters.drain()
guard snapshot.datagrams > 0 || snapshot.assembled > 0 else { continue }

let firstDatagrams = self.debugState.datagramsReceived == 0 && snapshot.datagrams > 0
let firstAssembled = self.debugState.assembledFrames == 0 && snapshot.assembled > 0
self.debugState.datagramsReceived += snapshot.datagrams
self.debugState.datagramBytesReceived += snapshot.datagramBytes
self.debugState.assembledFrames += snapshot.assembled
self.debugState.assembledFrameBytes += snapshot.assembledBytes
self.debugState.decodePackets += snapshot.parsed

if firstDatagrams {
self.record(
.info, "udp",
"UDP data flowing (\(snapshot.datagrams) datagrams, \(snapshot.datagramBytes) bytes in first batch)"
)
// Data is flowing — grant a fresh full window (once) so a slow/jittery
// network gets time to finish reassembly + decode instead of being cut
// off mid-handshake.
if self.state == .connecting && !self.didExtendTimeout {
self.didExtendTimeout = true
self.armConnectTimeout(seconds: Self.connectionTimeoutSeconds)

if snapshot.datagrams > 0 || snapshot.assembled > 0 {
let firstDatagrams =
self.debugState.datagramsReceived == 0 && snapshot.datagrams > 0
let firstAssembled =
self.debugState.assembledFrames == 0 && snapshot.assembled > 0
self.debugState.datagramsReceived += snapshot.datagrams
self.debugState.datagramBytesReceived += snapshot.datagramBytes
self.debugState.assembledFrames += snapshot.assembled
self.debugState.assembledFrameBytes += snapshot.assembledBytes
self.debugState.decodePackets += snapshot.parsed

if firstDatagrams {
self.record(
.info, "udp",
"UDP data flowing (\(snapshot.datagrams) datagrams, \(snapshot.datagramBytes) bytes in first batch)"
)
// Data is flowing — grant a fresh full window (once) so a slow/jittery
// network gets time to finish reassembly + decode instead of being cut
// off mid-handshake.
if self.state == .connecting && !self.didExtendTimeout {
self.didExtendTimeout = true
self.armConnectTimeout(seconds: Self.connectionTimeoutSeconds)
}
}
if firstAssembled {
self.record(.info, "assembly", "First frame payload assembled (\(snapshot.assembledBytes) bytes)")
}
}
if firstAssembled {
self.record(.info, "assembly", "First frame payload assembled (\(snapshot.assembledBytes) bytes)")
}

// Runs on EVERY tick, including silent ones: a dead host sends
// neither media nor heartbeats, so skipping this when nothing
// arrived is exactly when the liveness watchdog is needed.
self.refreshStatsAndWatchdog()
}
}
Expand All @@ -451,8 +458,17 @@ final class ConnectionManager: ObservableObject {
if let rtt = clock.rttUs { next.rttMs = Double(rtt) / 1000.0 }
if clock.offsetUs != nil { next.e2eMs = lagMs }
if let counters = frameAssembler?.counters.withLock({ $0 }) {
let deltaReceived = counters.fragsReceived &- prevCounters.fragsReceived
let deltaLost = counters.fragsLost &- prevCounters.fragsLost
// The assembler zeroes its counters whenever it resets (host
// pipeline restart bumps the stream epoch). A value BELOW last
// tick's therefore means "restarted", not "wrapped" — re-baseline,
// because subtracting through zero produced two ~2^64 deltas whose
// sum overflowed and killed the app on every host restart.
if counters.fragsReceived < prevCounters.fragsReceived
|| counters.fragsLost < prevCounters.fragsLost {
prevCounters = FrameAssembler.Counters()
}
let deltaReceived = counters.fragsReceived - prevCounters.fragsReceived
let deltaLost = counters.fragsLost - prevCounters.fragsLost
prevCounters = counters
let total = deltaReceived + deltaLost
next.lossPercent = total == 0 ? 0 : Double(deltaLost) * 100.0 / Double(total)
Expand Down Expand Up @@ -692,15 +708,33 @@ final class PixelBufferBox: @unchecked Sendable {

final class FrameSlot: @unchecked Sendable {
private let lock = os.OSAllocatedUnfairLock<PixelBufferBox?>(initialState: nil)

/// Holder so the redraw callback can be swapped under a lock. The Metal
/// view attaches and detaches it on the MainActor while the VideoToolbox
/// decode thread is calling it; an unguarded stored closure raced on its
/// own retain/release there, which crashes on connect and on teardown.
private struct CallbackHolder: @unchecked Sendable {
var callback: (() -> Void)?
}
private let stored = os.OSAllocatedUnfairLock<CallbackHolder>(
initialState: CallbackHolder()
)

/// Fired (on the storing thread — the VideoToolbox callback) every time a
/// frame lands, so the renderer can schedule an on-demand redraw. Assigned
/// once by the Metal view before streaming starts.
var onFrameStored: (() -> Void)?
/// frame lands, so the renderer can schedule an on-demand redraw.
var onFrameStored: (() -> Void)? {
get { stored.withLock { $0.callback } }
set { stored.withLock { $0.callback = newValue } }
}

func set(_ buffer: CVPixelBuffer) {
let box = PixelBufferBox(buffer)
lock.withLock { $0 = box }
onFrameStored?()
// Copy the closure out under the lock, then call it outside: holding
// the lock across arbitrary caller code invites deadlock, and a local
// strong reference keeps a concurrent detach from freeing it mid-call.
let callback = stored.withLock { $0.callback }
callback?()
}

func take() -> CVPixelBuffer? {
Expand Down
29 changes: 16 additions & 13 deletions ios/EternalMonitor/App/DisplayView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -18,25 +18,15 @@ struct DisplayView: View {
// PC; a three-finger tap (via the relay layer) drives the HUD.
MetalView()
.ignoresSafeArea()
TouchRelayView(onToggleHUD: {
showHUD.toggle()
scheduleHUDDismiss()
})
.ignoresSafeArea()
TouchRelayView(onToggleHUD: { toggleHUD() })
.ignoresSafeArea()
} else {
// One gesture, one meaning: tap toggles the HUD. (The old
// single-tap-hide + triple-tap-toggle pair made every triple
// tap race its own first tap.)
MetalView()
.ignoresSafeArea()
.onTapGesture {
if showHUD {
hudDismissTask?.cancel()
withAnimation(.easeOut(duration: 0.25)) { showHUD = false }
} else {
scheduleHUDDismiss()
}
}
.onTapGesture { toggleHUD() }
}

// Viewfinder registration marks frame the picture (fade with the HUD).
Expand Down Expand Up @@ -236,6 +226,19 @@ struct DisplayView: View {

// MARK: - Auto-hide HUD

/// Show the HUD (auto-hiding after a few seconds) or hide it now. Both the
/// plain tap and the three-finger relay tap route here: calling
/// `scheduleHUDDismiss()` to "toggle" could only ever show it, since that
/// function unconditionally sets `showHUD = true`.
private func toggleHUD() {
if showHUD {
hudDismissTask?.cancel()
withAnimation(.easeOut(duration: 0.25)) { showHUD = false }
} else {
scheduleHUDDismiss()
}
}

private func scheduleHUDDismiss() {
hudDismissTask?.cancel()
withAnimation(.easeOut(duration: 0.25)) { showHUD = true }
Expand Down
2 changes: 1 addition & 1 deletion ios/EternalMonitor/Info.plist
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<true/>
<key>UIAppFonts</key>
<array>
<string>Syne-Variable.ttf</string>
<string>Syne-Bold.ttf</string>
<string>JetBrainsMono-Regular.ttf</string>
<string>JetBrainsMono-Medium.ttf</string>
</array>
Expand Down
9 changes: 6 additions & 3 deletions ios/EternalMonitor/Input/TouchRelay.swift
Original file line number Diff line number Diff line change
Expand Up @@ -177,9 +177,12 @@ struct TouchRelayMachine {
switch mode {
case .pending(let start, _):
guard let point else { return [] }
let dx = Int32(point.x) - Int32(start.x)
let dy = Int32(point.y) - Int32(start.y)
if dx * dx + dy * dy > Self.dragSlop * Self.dragSlop {
// Int64: the coordinates span 0...65535, so squaring a large flick
// overflows Int32 and Swift traps rather than wrapping.
let dx = Int64(point.x) - Int64(start.x)
let dy = Int64(point.y) - Int64(start.y)
let slop = Int64(Self.dragSlop)
if dx * dx + dy * dy > slop * slop {
// Committed to a drag: press at the start point, catch up.
mode = .leftDown(isPencil: false)
var outputs = edge(Phase.began, kind: Kind.touch, buttons: 1, at: start, timeUs: timeUs)
Expand Down
12 changes: 8 additions & 4 deletions ios/EternalMonitor/Network/FrameAssembler.swift
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,14 @@ final class FrameAssembler {
var onFrameAssembled: ((Data, _ seq: UInt32, _ captureTimestampUs: UInt64, _ isKeyframe: Bool) -> Void)?
var onDiagnostic: ((String) -> Void)?

/// A frame may span at most this many fragments (~1.4 MB at 1384-byte
/// payloads). The u16 field allows 65535 (≈90 MB) — a memory bomb, not a
/// video frame.
static let maxFragmentCount: UInt16 = 1024
/// A frame may span at most this many fragments. This MUST match the
/// protocol cap the host fragments against and the wire parser enforces:
/// a tighter value here silently discards legitimately large access units
/// (a 1440p/4K scene-change IDR runs past 1024 fragments), and because the
/// decoder then never receives a sync sample, every later frame is dropped
/// too — a permanent freeze with healthy heartbeats. `maxPendingBytes` is
/// the real memory guard; the u16 field alone would allow ≈90 MB.
static let maxFragmentCount: UInt16 = MediaHeader.maxFragCount
/// At most this many partial frames in flight; the oldest is dropped first.
static let maxPendingFrames = 8
/// Hard ceiling on buffered fragment bytes across all partial frames.
Expand Down
19 changes: 19 additions & 0 deletions ios/EternalMonitorTests/FrameAssemblerTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,25 @@ final class FrameAssemblerTests: XCTestCase {
XCTAssertEqual(completed.count, 2, "stale-epoch fragment must be dropped")
}

func testAcceptsFramesUpToTheProtocolFragmentCap() {
// A 1440p/4K scene-change IDR runs past 1024 fragments. The host
// fragments against the protocol cap and the wire parser accepts up to
// it, so an assembler cap below that silently dropped every fragment
// of such a frame — and with no sync sample the decoder then discarded
// everything after it too.
XCTAssertEqual(
FrameAssembler.maxFragmentCount, MediaHeader.maxFragCount,
"the assembler cap must match the protocol cap the host sends against"
)

let count: UInt16 = 2000
for index in 0..<count {
add(seq: 1, index: index, count: count, byte: 0xA)
}
XCTAssertEqual(completed.count, 1, "a large but legal frame must assemble")
XCTAssertEqual(completed.first?.count, Int(count))
}

func testOneBogusEpochCannotStrandTheStream() {
add(seq: 10, index: 0, count: 1, epoch: 5, byte: 0xA)
XCTAssertEqual(completed.count, 1)
Expand Down
12 changes: 12 additions & 0 deletions ios/EternalMonitorTests/TouchRelayTests.swift
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,18 @@ final class TouchRelayMachineTests: XCTestCase {
XCTAssertEqual(release[0].phase, TouchRelayMachine.Phase.ended)
}

func testFullScreenFlickDoesNotOverflow() {
// Coordinates span 0...65535, so squaring the delta of a large flick
// overflows Int32 — and Swift traps rather than wrapping, killing the
// app mid-gesture. One corner to the other is the worst case.
_ = machine.touchBegan(at: P(x: 0, y: 0), isPencil: false, timeUs: t)
let outputs = sends(machine.touchMoved(
to: P(x: 65535, y: 65535), centroid: nil, isPencil: false, force: 0, timeUs: t + 20_000
))
XCTAssertEqual(outputs.count, 3, "a flick past the slop commits a drag")
XCTAssertEqual(outputs[0].phase, TouchRelayMachine.Phase.began)
}

func testSmallJitterStaysATap() {
let start = P(x: 10000, y: 10000)
_ = machine.touchBegan(at: start, isPencil: false, timeUs: t)
Expand Down
Loading