Auto-downgrade to MJPEG when VideoToolbox can't encode H.264

packages/serve-sim-client/src/__tests__/avcc-codec.test.ts

@@ -6,6 +6,7 @@ import {
   AVCC_TAG_KEYFRAME,
   AVCC_TAG_DELTA,
   AVCC_TAG_SEED,
+  AVCC_TAG_DOWNGRADE,
 } from "../avcc-codec";
 
 /** Build one wire chunk: [len:u32-be][tag][payload]. len = payload + 1. */
@@ -98,6 +99,15 @@ describe("AvccDemuxer", () => {
     expect(Array.from(rest[0]!.payload)).toEqual([2, 3]);
   });
 
+  test("parses the empty-payload downgrade signal", () => {
+    const d = new AvccDemuxer();
+    const chunks = d.push(
+      concat(frame(AVCC_TAG_SEED, [0xff, 0xd8]), frame(AVCC_TAG_DOWNGRADE, [])),
+    );
+    expect(chunks.map((c) => c.type)).toEqual(["seed", "downgrade"]);
+    expect(chunks[1]!.payload).toHaveLength(0);
+  });
+
   test("skips unknown tags without stalling the stream", () => {
     const d = new AvccDemuxer();
     const chunks = d.push(

packages/serve-sim-client/src/avcc-codec.ts

@@ -11,6 +11,9 @@
  *   0x02 keyframe    — IDR (decodable standalone)
  *   0x03 delta       — non-IDR P-frame
  *   0x04 seed        — JPEG painted before the first IDR decodes
+ *   0x05 downgrade   — the helper's VideoToolbox encoder can't produce H.264
+ *                      (e.g. virtualized macOS); the viewer should drop to MJPEG
+ *                      now instead of waiting out the no-frame timeout
  *
  * The stream is read incrementally from a `fetch()` ReadableStream, so chunks
  * arrive split across reads. `AvccDemuxer` buffers partial bytes and yields
@@ -21,8 +24,9 @@ export const AVCC_TAG_DESCRIPTION = 0x01;
 export const AVCC_TAG_KEYFRAME = 0x02;
 export const AVCC_TAG_DELTA = 0x03;
 export const AVCC_TAG_SEED = 0x04;
+export const AVCC_TAG_DOWNGRADE = 0x05;
 
-export type AvccChunkType = "description" | "keyframe" | "delta" | "seed";
+export type AvccChunkType = "description" | "keyframe" | "delta" | "seed" | "downgrade";
 
 export interface AvccChunk {
   type: AvccChunkType;
@@ -35,6 +39,7 @@ const TAG_TO_TYPE: Record<number, AvccChunkType | undefined> = {
   [AVCC_TAG_KEYFRAME]: "keyframe",
   [AVCC_TAG_DELTA]: "delta",
   [AVCC_TAG_SEED]: "seed",
+  [AVCC_TAG_DOWNGRADE]: "downgrade",
 };
 
 /**

packages/serve-sim-client/src/simulator/use-avcc-stream.ts

@@ -162,6 +162,12 @@ export function useAvccStream({
         case "delta":
           decodeFrame(type, payload);
           return;
+        case "downgrade":
+          // The helper reports its VideoToolbox encoder can't produce H.264
+          // (virtualized macOS). Downgrade to MJPEG immediately rather than
+          // sitting on the frozen seed until the no-frame timeout fires.
+          reportDecodeFailure("server: H.264 encode unavailable");
+          return;
       }
     };
 

packages/serve-sim/Sources/SimStreamHelper/ClientManager.swift

@@ -25,6 +25,12 @@ final class ClientManager {
     /// the new decoder needs an IDR before any delta will decode.
     var onAvccClientConnect: (() -> Void)?
 
+    /// Whether VideoToolbox H.264 encode actually produces output on this host.
+    /// `nil` until the startup probe decides; `false` on virtualized macOS where
+    /// encode silently no-ops. When `false`, AVCC viewers are told to downgrade
+    /// to MJPEG immediately instead of stalling on a frozen seed.
+    private var h264Supported: Bool?
+
     var onTouch: ((TouchEventPayload) -> Void)?
     var onButton: ((String) -> Void)?
     /// Arbitrary HID hardware button by (page, usage, phase) — power / volume /
@@ -60,11 +66,35 @@ final class ClientManager {
     func screenConfig() -> [String: Any] {
         configLock.lock()
         defer { configLock.unlock() }
-        return [
+        var config: [String: Any] = [
             "width": screenWidth,
             "height": screenHeight,
             "orientation": screenOrientation,
         ]
+        // Surface the probe result once known so the preview/diagnostics can
+        // see why a stream is on MJPEG (omitted while still undecided).
+        if let h264Supported { config["h264Supported"] = h264Supported }
+        return config
+    }
+
+    /// Record the startup H.264 probe result. When encode is unavailable, tell
+    /// any already-connected AVCC viewers to downgrade to MJPEG right away.
+    func setH264Supported(_ supported: Bool) {
+        configLock.lock()
+        let changed = h264Supported != supported
+        h264Supported = supported
+        configLock.unlock()
+        guard changed else { return }
+        if !supported {
+            broadcastAvcc(AVCCEnvelope.downgrade())
+        }
+        broadcastConfig()
+    }
+
+    private func h264SupportedSnapshot() -> Bool? {
+        configLock.lock()
+        defer { configLock.unlock() }
+        return h264Supported
     }
 
     /// Tag for a server->client screen-config push. Distinct from the
@@ -147,15 +177,25 @@ final class ClientManager {
     /// replay the cached decoder description, then ask the owner to force a
     /// keyframe so an IDR follows promptly.
     func sendInitialAvcc(to client: AVCCClient) {
+        let supported = h264SupportedSnapshot()
         queue.async {
             if let jpeg = self.latestFrame {
                 client.send(AVCCEnvelope.seed(jpeg: jpeg))
             }
+            // Encoder already known dead — paint the seed, then send the viewer
+            // straight to MJPEG without waiting for an IDR that won't come.
+            if supported == false {
+                client.send(AVCCEnvelope.downgrade())
+                return
+            }
             if let desc = self.cachedAvccDescription {
                 client.send(desc)
             }
         }
-        onAvccClientConnect?()
+        // Only bother forcing a keyframe when H.264 might actually work.
+        if supported != false {
+            onAvccClientConnect?()
+        }
     }
 
     func removeAvccClient(_ client: AVCCClient) {

packages/serve-sim/Sources/SimStreamHelper/StreamFormat.swift

@@ -29,11 +29,16 @@ enum AVCCEnvelope {
     static let keyframeTag: UInt8 = 0x02
     static let deltaTag: UInt8 = 0x03
     static let seedTag: UInt8 = 0x04
+    // Empty-payload signal: VideoToolbox H.264 encode is unavailable on this
+    // host (e.g. virtualized macOS), so the viewer should switch to MJPEG now
+    // instead of waiting out its no-frame timeout.
+    static let downgradeTag: UInt8 = 0x05
 
     static func description(avcc: Data) -> Data { wrap(tag: descriptionTag, payload: avcc) }
     static func keyframe(avcc: Data) -> Data { wrap(tag: keyframeTag, payload: avcc) }
     static func delta(avcc: Data) -> Data { wrap(tag: deltaTag, payload: avcc) }
     static func seed(jpeg: Data) -> Data { wrap(tag: seedTag, payload: jpeg) }
+    static func downgrade() -> Data { wrap(tag: downgradeTag, payload: Data()) }
 
     private static func wrap(tag: UInt8, payload: Data) -> Data {
         let length = UInt32(payload.count + 1)

packages/serve-sim/Sources/SimStreamHelper/main.swift

@@ -56,8 +56,29 @@ var h264Encoding = false // backpressure flag (H.264)
 // so the freshly-configured decoder has a keyframe to start from.
 var forceKeyframe = false
 
+// ─── H.264 capability probe ───
+// VideoToolbox H.264 encode silently no-ops on some virtualized macOS hosts:
+// `VTCompressionSessionCreate` succeeds but no encoded frame is ever produced.
+// At startup we feed a few captured frames through the encoder — independent of
+// any AVCC viewer — and watch for output, so the preview can default to MJPEG
+// immediately instead of stalling on a frozen seed. All probe state is touched
+// only on `h264Queue`.
+var h264Decided = false       // true once support is known either way
+var h264OK = false            // decided AND VideoToolbox actually encodes
+var h264ProbeStart: Date?     // first probe frame's timestamp
+let h264ProbeTimeout: TimeInterval = 3.0
+
 // H.264 output → AVCC envelope → broadcast to /stream.avcc clients.
 h264Encoder.onEncoded = { encoded in
+    // First successful output proves VideoToolbox H.264 works on this host.
+    h264Queue.async {
+        if !h264Decided {
+            h264Decided = true
+            h264OK = true
+            print("[h264] probe: VideoToolbox H.264 encode OK")
+            httpServer.clientManager.setH264Supported(true)
+        }
+    }
     if let description = encoded.description {
         httpServer.clientManager.broadcastAvcc(AVCCEnvelope.description(avcc: description), isDescription: true)
     }
@@ -165,19 +186,44 @@ let frameHandler: (CVPixelBuffer, CMTime) -> Void = { pixelBuffer, timestamp in
         }
     }
 
-    // H.264 path runs only while at least one AVCC viewer is connected, so an
-    // all-MJPEG session pays no VideoToolbox cost. Its own backpressure flag
-    // lets it skip independently of the JPEG encoder.
-    if httpServer.clientManager.hasAvccClients() {
-        h264Queue.async {
-            if h264Encoding { return }
-            h264Encoding = true
-            let force = forceKeyframe
-            forceKeyframe = false
-            h264Encoder.encode(pixelBuffer, forceKeyframe: force) {
-                h264Queue.async {
-                    h264Encoding = false
-                }
+    // H.264 path runs while an AVCC viewer is connected (so an all-MJPEG
+    // session pays no VideoToolbox cost) OR while the startup probe is still
+    // deciding whether this host can encode H.264 at all. All probe/backpressure
+    // state (`h264Decided`, `h264OK`, `h264ProbeStart`, `h264Encoding`,
+    // `forceKeyframe`) is confined to `h264Queue`, so the gating decision is made
+    // inside the queue rather than read racily from the capture thread.
+    h264Queue.async {
+        let hasAvccClients = httpServer.clientManager.hasAvccClients()
+        guard hasAvccClients || !h264Decided else { return }
+
+        if !h264Decided {
+            // Drive the one-shot probe: force the first frame to a keyframe so
+            // VideoToolbox produces an IDR fast, and give up (→ MJPEG) if nothing
+            // comes back within the timeout. This decision runs before the
+            // backpressure check below so a wedged encode can't suppress it.
+            if h264ProbeStart == nil {
+                h264ProbeStart = Date()
+                forceKeyframe = true
+            } else if Date().timeIntervalSince(h264ProbeStart!) > h264ProbeTimeout {
+                h264Decided = true
+                h264OK = false
+                print("[h264] probe: no VideoToolbox output in \(Int(h264ProbeTimeout))s — viewers will use MJPEG")
+                httpServer.clientManager.setH264Supported(false)
+                return
+            }
+        } else if !h264OK || !hasAvccClients {
+            // Decided unsupported, or no viewers: nothing to encode.
+            return
+        }
+
+        // Backpressure: skip this frame if the previous encode is still in flight.
+        if h264Encoding { return }
+        h264Encoding = true
+        let force = forceKeyframe
+        forceKeyframe = false
+        h264Encoder.encode(pixelBuffer, forceKeyframe: force) {
+            h264Queue.async {
+                h264Encoding = false
             }
         }
     }

packages/serve-sim/src/__tests__/avcc-stream-endpoint.test.ts

@@ -20,6 +20,8 @@ const TAG_DESCRIPTION = 0x01;
 const TAG_KEYFRAME = 0x02;
 const TAG_DELTA = 0x03;
 const TAG_SEED = 0x04;
+const TAG_DOWNGRADE = 0x05;
+const VALID_TAGS = [TAG_DESCRIPTION, TAG_KEYFRAME, TAG_DELTA, TAG_SEED, TAG_DOWNGRADE];
 
 function firstBootedIosSim(): string | null {
   try {
@@ -82,6 +84,10 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "<skipped>"
   test("emits a decoder description and a keyframe", async () => {
     const controller = new AbortController();
     const timer = setTimeout(() => controller.abort(), STREAM_BUDGET_MS);
+    // Armed once a downgrade is seen: keep reading briefly so a keyframe emitted
+    // *after* downgrade (a protocol violation) is still observed before we
+    // assert none exist, then end the read.
+    let downgradeGrace: ReturnType<typeof setTimeout> | null = null;
 
     const seenTags = new Set<number>();
     let buffer = new Uint8Array(0);
@@ -108,17 +114,36 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "<skipped>"
             consumedBytes += envelope.consumed;
           }
           if (consumedBytes > 0) buffer = buffer.subarray(consumedBytes);
-          // Stop as soon as we've proven a decodable stream: config + an IDR.
+          // Stop as soon as we've proven a decodable stream (config + an IDR).
           if (seenTags.has(TAG_DESCRIPTION) && seenTags.has(TAG_KEYFRAME)) break;
+          // On downgrade the helper says H.264 is unavailable. Don't break
+          // immediately — keep reading for a short grace window so a stray
+          // post-downgrade keyframe would be caught by the assertion below.
+          if (seenTags.has(TAG_DOWNGRADE) && downgradeGrace === null) {
+            clearTimeout(timer);
+            downgradeGrace = setTimeout(() => controller.abort(), 300);
+          }
         }
       }
     } catch (e) {
       if ((e as Error).name !== "AbortError") throw e;
     } finally {
       clearTimeout(timer);
+      if (downgradeGrace) clearTimeout(downgradeGrace);
       controller.abort();
     }
 
+    // The helper probes VideoToolbox at startup and emits a downgrade envelope
+    // when it can't encode H.264 (e.g. a virtualized macOS runner). That's the
+    // designed behavior — assert valid framing and that it didn't also claim a
+    // real H.264 keyframe, then pass. This deterministically covers the VM path
+    // instead of leaning on the timeout-based soft-pass below.
+    if (seenTags.has(TAG_DOWNGRADE)) {
+      for (const tag of seenTags) expect(VALID_TAGS).toContain(tag);
+      expect(seenTags.has(TAG_KEYFRAME)).toBe(false);
+      return;
+    }
+
     const decodable = seenTags.has(TAG_DESCRIPTION) && seenTags.has(TAG_KEYFRAME);
 
     // VideoToolbox's H.264 encoder frequently fails to warm on GitHub macOS
@@ -139,7 +164,7 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "<skipped>"
       );
       // Whatever did arrive must still be valid envelope framing.
       for (const tag of seenTags) {
-        expect([TAG_DESCRIPTION, TAG_KEYFRAME, TAG_DELTA, TAG_SEED]).toContain(tag);
+        expect(VALID_TAGS).toContain(tag);
       }
       return;
     }
@@ -149,7 +174,7 @@ describeWithSim(`serve-sim AVCC endpoint (booted sim ${bootedUdid ?? "<skipped>"
     expect(seenTags.has(TAG_DESCRIPTION)).toBe(true);
     expect(seenTags.has(TAG_KEYFRAME)).toBe(true);
     for (const tag of seenTags) {
-      expect([TAG_DESCRIPTION, TAG_KEYFRAME, TAG_DELTA, TAG_SEED]).toContain(tag);
+      expect(VALID_TAGS).toContain(tag);
     }
   }, STREAM_BUDGET_MS + 5_000);
 });