diff --git a/packages/serve-sim-client/src/__tests__/avcc-codec.test.ts b/packages/serve-sim-client/src/__tests__/avcc-codec.test.ts index 9f9e6425..62d218fd 100644 --- a/packages/serve-sim-client/src/__tests__/avcc-codec.test.ts +++ b/packages/serve-sim-client/src/__tests__/avcc-codec.test.ts @@ -6,6 +6,7 @@ import { AVCC_TAG_KEYFRAME, AVCC_TAG_DELTA, AVCC_TAG_SEED, + AVCC_TAG_DOWNGRADE, } from "../avcc-codec"; /** Build one wire chunk: [len:u32-be][tag][payload]. len = payload + 1. */ @@ -98,6 +99,15 @@ describe("AvccDemuxer", () => { expect(Array.from(rest[0]!.payload)).toEqual([2, 3]); }); + test("parses the empty-payload downgrade signal", () => { + const d = new AvccDemuxer(); + const chunks = d.push( + concat(frame(AVCC_TAG_SEED, [0xff, 0xd8]), frame(AVCC_TAG_DOWNGRADE, [])), + ); + expect(chunks.map((c) => c.type)).toEqual(["seed", "downgrade"]); + expect(chunks[1]!.payload).toHaveLength(0); + }); + test("skips unknown tags without stalling the stream", () => { const d = new AvccDemuxer(); const chunks = d.push( diff --git a/packages/serve-sim-client/src/avcc-codec.ts b/packages/serve-sim-client/src/avcc-codec.ts index 4eb7dbac..9ff69fcd 100644 --- a/packages/serve-sim-client/src/avcc-codec.ts +++ b/packages/serve-sim-client/src/avcc-codec.ts @@ -11,6 +11,9 @@ * 0x02 keyframe — IDR (decodable standalone) * 0x03 delta — non-IDR P-frame * 0x04 seed — JPEG painted before the first IDR decodes + * 0x05 downgrade — the helper's VideoToolbox encoder can't produce H.264 + * (e.g. virtualized macOS); the viewer should drop to MJPEG + * now instead of waiting out the no-frame timeout * * The stream is read incrementally from a `fetch()` ReadableStream, so chunks * arrive split across reads. `AvccDemuxer` buffers partial bytes and yields @@ -21,8 +24,9 @@ export const AVCC_TAG_DESCRIPTION = 0x01; export const AVCC_TAG_KEYFRAME = 0x02; export const AVCC_TAG_DELTA = 0x03; export const AVCC_TAG_SEED = 0x04; +export const AVCC_TAG_DOWNGRADE = 0x05; -export type AvccChunkType = "description" | "keyframe" | "delta" | "seed"; +export type AvccChunkType = "description" | "keyframe" | "delta" | "seed" | "downgrade"; export interface AvccChunk { type: AvccChunkType; @@ -35,6 +39,7 @@ const TAG_TO_TYPE: Record = { [AVCC_TAG_KEYFRAME]: "keyframe", [AVCC_TAG_DELTA]: "delta", [AVCC_TAG_SEED]: "seed", + [AVCC_TAG_DOWNGRADE]: "downgrade", }; /** diff --git a/packages/serve-sim-client/src/simulator/use-avcc-stream.ts b/packages/serve-sim-client/src/simulator/use-avcc-stream.ts index 8cc5ee56..89728604 100644 --- a/packages/serve-sim-client/src/simulator/use-avcc-stream.ts +++ b/packages/serve-sim-client/src/simulator/use-avcc-stream.ts @@ -162,6 +162,12 @@ export function useAvccStream({ case "delta": decodeFrame(type, payload); return; + case "downgrade": + // The helper reports its VideoToolbox encoder can't produce H.264 + // (virtualized macOS). Downgrade to MJPEG immediately rather than + // sitting on the frozen seed until the no-frame timeout fires. + reportDecodeFailure("server: H.264 encode unavailable"); + return; } }; diff --git a/packages/serve-sim/Sources/SimStreamHelper/ClientManager.swift b/packages/serve-sim/Sources/SimStreamHelper/ClientManager.swift index dba70d65..06bca146 100644 --- a/packages/serve-sim/Sources/SimStreamHelper/ClientManager.swift +++ b/packages/serve-sim/Sources/SimStreamHelper/ClientManager.swift @@ -25,6 +25,12 @@ final class ClientManager { /// the new decoder needs an IDR before any delta will decode. var onAvccClientConnect: (() -> Void)? + /// Whether VideoToolbox H.264 encode actually produces output on this host. + /// `nil` until the startup probe decides; `false` on virtualized macOS where + /// encode silently no-ops. When `false`, AVCC viewers are told to downgrade + /// to MJPEG immediately instead of stalling on a frozen seed. + private var h264Supported: Bool? + var onTouch: ((TouchEventPayload) -> Void)? var onButton: ((String) -> Void)? /// Arbitrary HID hardware button by (page, usage, phase) — power / volume / @@ -60,11 +66,35 @@ final class ClientManager { func screenConfig() -> [String: Any] { configLock.lock() defer { configLock.unlock() } - return [ + var config: [String: Any] = [ "width": screenWidth, "height": screenHeight, "orientation": screenOrientation, ] + // Surface the probe result once known so the preview/diagnostics can + // see why a stream is on MJPEG (omitted while still undecided). + if let h264Supported { config["h264Supported"] = h264Supported } + return config + } + + /// Record the startup H.264 probe result. When encode is unavailable, tell + /// any already-connected AVCC viewers to downgrade to MJPEG right away. + func setH264Supported(_ supported: Bool) { + configLock.lock() + let changed = h264Supported != supported + h264Supported = supported + configLock.unlock() + guard changed else { return } + if !supported { + broadcastAvcc(AVCCEnvelope.downgrade()) + } + broadcastConfig() + } + + private func h264SupportedSnapshot() -> Bool? { + configLock.lock() + defer { configLock.unlock() } + return h264Supported } /// Tag for a server->client screen-config push. Distinct from the @@ -147,15 +177,25 @@ final class ClientManager { /// replay the cached decoder description, then ask the owner to force a /// keyframe so an IDR follows promptly. func sendInitialAvcc(to client: AVCCClient) { + let supported = h264SupportedSnapshot() queue.async { if let jpeg = self.latestFrame { client.send(AVCCEnvelope.seed(jpeg: jpeg)) } + // Encoder already known dead — paint the seed, then send the viewer + // straight to MJPEG without waiting for an IDR that won't come. + if supported == false { + client.send(AVCCEnvelope.downgrade()) + return + } if let desc = self.cachedAvccDescription { client.send(desc) } } - onAvccClientConnect?() + // Only bother forcing a keyframe when H.264 might actually work. + if supported != false { + onAvccClientConnect?() + } } func removeAvccClient(_ client: AVCCClient) { diff --git a/packages/serve-sim/Sources/SimStreamHelper/StreamFormat.swift b/packages/serve-sim/Sources/SimStreamHelper/StreamFormat.swift index 29ec48de..d1d3afc8 100644 --- a/packages/serve-sim/Sources/SimStreamHelper/StreamFormat.swift +++ b/packages/serve-sim/Sources/SimStreamHelper/StreamFormat.swift @@ -29,11 +29,16 @@ enum AVCCEnvelope { static let keyframeTag: UInt8 = 0x02 static let deltaTag: UInt8 = 0x03 static let seedTag: UInt8 = 0x04 + // Empty-payload signal: VideoToolbox H.264 encode is unavailable on this + // host (e.g. virtualized macOS), so the viewer should switch to MJPEG now + // instead of waiting out its no-frame timeout. + static let downgradeTag: UInt8 = 0x05 static func description(avcc: Data) -> Data { wrap(tag: descriptionTag, payload: avcc) } static func keyframe(avcc: Data) -> Data { wrap(tag: keyframeTag, payload: avcc) } static func delta(avcc: Data) -> Data { wrap(tag: deltaTag, payload: avcc) } static func seed(jpeg: Data) -> Data { wrap(tag: seedTag, payload: jpeg) } + static func downgrade() -> Data { wrap(tag: downgradeTag, payload: Data()) } private static func wrap(tag: UInt8, payload: Data) -> Data { let length = UInt32(payload.count + 1) diff --git a/packages/serve-sim/Sources/SimStreamHelper/main.swift b/packages/serve-sim/Sources/SimStreamHelper/main.swift index d3632603..d8b80498 100644 --- a/packages/serve-sim/Sources/SimStreamHelper/main.swift +++ b/packages/serve-sim/Sources/SimStreamHelper/main.swift @@ -56,8 +56,29 @@ var h264Encoding = false // backpressure flag (H.264) // so the freshly-configured decoder has a keyframe to start from. var forceKeyframe = false +// ─── H.264 capability probe ─── +// VideoToolbox H.264 encode silently no-ops on some virtualized macOS hosts: +// `VTCompressionSessionCreate` succeeds but no encoded frame is ever produced. +// At startup we feed a few captured frames through the encoder — independent of +// any AVCC viewer — and watch for output, so the preview can default to MJPEG +// immediately instead of stalling on a frozen seed. All probe state is touched +// only on `h264Queue`. +var h264Decided = false // true once support is known either way +var h264OK = false // decided AND VideoToolbox actually encodes +var h264ProbeStart: Date? // first probe frame's timestamp +let h264ProbeTimeout: TimeInterval = 3.0 + // H.264 output → AVCC envelope → broadcast to /stream.avcc clients. h264Encoder.onEncoded = { encoded in + // First successful output proves VideoToolbox H.264 works on this host. + h264Queue.async { + if !h264Decided { + h264Decided = true + h264OK = true + print("[h264] probe: VideoToolbox H.264 encode OK") + httpServer.clientManager.setH264Supported(true) + } + } if let description = encoded.description { httpServer.clientManager.broadcastAvcc(AVCCEnvelope.description(avcc: description), isDescription: true) } @@ -165,19 +186,44 @@ let frameHandler: (CVPixelBuffer, CMTime) -> Void = { pixelBuffer, timestamp in } } - // H.264 path runs only while at least one AVCC viewer is connected, so an - // all-MJPEG session pays no VideoToolbox cost. Its own backpressure flag - // lets it skip independently of the JPEG encoder. - if httpServer.clientManager.hasAvccClients() { - h264Queue.async { - if h264Encoding { return } - h264Encoding = true - let force = forceKeyframe - forceKeyframe = false - h264Encoder.encode(pixelBuffer, forceKeyframe: force) { - h264Queue.async { - h264Encoding = false - } + // H.264 path runs while an AVCC viewer is connected (so an all-MJPEG + // session pays no VideoToolbox cost) OR while the startup probe is still + // deciding whether this host can encode H.264 at all. All probe/backpressure + // state (`h264Decided`, `h264OK`, `h264ProbeStart`, `h264Encoding`, + // `forceKeyframe`) is confined to `h264Queue`, so the gating decision is made + // inside the queue rather than read racily from the capture thread. + h264Queue.async { + let hasAvccClients = httpServer.clientManager.hasAvccClients() + guard hasAvccClients || !h264Decided else { return } + + if !h264Decided { + // Drive the one-shot probe: force the first frame to a keyframe so + // VideoToolbox produces an IDR fast, and give up (→ MJPEG) if nothing + // comes back within the timeout. This decision runs before the + // backpressure check below so a wedged encode can't suppress it. + if h264ProbeStart == nil { + h264ProbeStart = Date() + forceKeyframe = true + } else if Date().timeIntervalSince(h264ProbeStart!) > h264ProbeTimeout { + h264Decided = true + h264OK = false + print("[h264] probe: no VideoToolbox output in \(Int(h264ProbeTimeout))s — viewers will use MJPEG") + httpServer.clientManager.setH264Supported(false) + return + } + } else if !h264OK || !hasAvccClients { + // Decided unsupported, or no viewers: nothing to encode. + return + } + + // Backpressure: skip this frame if the previous encode is still in flight. + if h264Encoding { return } + h264Encoding = true + let force = forceKeyframe + forceKeyframe = false + h264Encoder.encode(pixelBuffer, forceKeyframe: force) { + h264Queue.async { + h264Encoding = false } } } diff --git a/packages/serve-sim/bin/serve-sim-bin b/packages/serve-sim/bin/serve-sim-bin index a626bafc..37065687 100755 Binary files a/packages/serve-sim/bin/serve-sim-bin and b/packages/serve-sim/bin/serve-sim-bin differ diff --git a/packages/serve-sim/src/__tests__/avcc-stream-endpoint.test.ts b/packages/serve-sim/src/__tests__/avcc-stream-endpoint.test.ts index a92c4c20..85a3e594 100644 --- a/packages/serve-sim/src/__tests__/avcc-stream-endpoint.test.ts +++ b/packages/serve-sim/src/__tests__/avcc-stream-endpoint.test.ts @@ -20,6 +20,8 @@ const TAG_DESCRIPTION = 0x01; const TAG_KEYFRAME = 0x02; const TAG_DELTA = 0x03; const TAG_SEED = 0x04; +const TAG_DOWNGRADE = 0x05; +const VALID_TAGS = [TAG_DESCRIPTION, TAG_KEYFRAME, TAG_DELTA, TAG_SEED, TAG_DOWNGRADE]; function firstBootedIosSim(): string | null { try { @@ -82,6 +84,10 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "" test("emits a decoder description and a keyframe", async () => { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), STREAM_BUDGET_MS); + // Armed once a downgrade is seen: keep reading briefly so a keyframe emitted + // *after* downgrade (a protocol violation) is still observed before we + // assert none exist, then end the read. + let downgradeGrace: ReturnType | null = null; const seenTags = new Set(); let buffer = new Uint8Array(0); @@ -108,17 +114,36 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "" consumedBytes += envelope.consumed; } if (consumedBytes > 0) buffer = buffer.subarray(consumedBytes); - // Stop as soon as we've proven a decodable stream: config + an IDR. + // Stop as soon as we've proven a decodable stream (config + an IDR). if (seenTags.has(TAG_DESCRIPTION) && seenTags.has(TAG_KEYFRAME)) break; + // On downgrade the helper says H.264 is unavailable. Don't break + // immediately — keep reading for a short grace window so a stray + // post-downgrade keyframe would be caught by the assertion below. + if (seenTags.has(TAG_DOWNGRADE) && downgradeGrace === null) { + clearTimeout(timer); + downgradeGrace = setTimeout(() => controller.abort(), 300); + } } } } catch (e) { if ((e as Error).name !== "AbortError") throw e; } finally { clearTimeout(timer); + if (downgradeGrace) clearTimeout(downgradeGrace); controller.abort(); } + // The helper probes VideoToolbox at startup and emits a downgrade envelope + // when it can't encode H.264 (e.g. a virtualized macOS runner). That's the + // designed behavior — assert valid framing and that it didn't also claim a + // real H.264 keyframe, then pass. This deterministically covers the VM path + // instead of leaning on the timeout-based soft-pass below. + if (seenTags.has(TAG_DOWNGRADE)) { + for (const tag of seenTags) expect(VALID_TAGS).toContain(tag); + expect(seenTags.has(TAG_KEYFRAME)).toBe(false); + return; + } + const decodable = seenTags.has(TAG_DESCRIPTION) && seenTags.has(TAG_KEYFRAME); // VideoToolbox's H.264 encoder frequently fails to warm on GitHub macOS @@ -139,7 +164,7 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "" ); // Whatever did arrive must still be valid envelope framing. for (const tag of seenTags) { - expect([TAG_DESCRIPTION, TAG_KEYFRAME, TAG_DELTA, TAG_SEED]).toContain(tag); + expect(VALID_TAGS).toContain(tag); } return; } @@ -149,7 +174,7 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "" expect(seenTags.has(TAG_DESCRIPTION)).toBe(true); expect(seenTags.has(TAG_KEYFRAME)).toBe(true); for (const tag of seenTags) { - expect([TAG_DESCRIPTION, TAG_KEYFRAME, TAG_DELTA, TAG_SEED]).toContain(tag); + expect(VALID_TAGS).toContain(tag); } }, STREAM_BUDGET_MS + 5_000); });