From ed6b2da643e5c16c315c4e4a93f0ac601c5cb32c Mon Sep 17 00:00:00 2001 From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com> Date: Thu, 12 Nov 2026 23:05:31 -0500 Subject: [PATCH] Add MPEG-2 and H.263+ codec negotiation --- src/Limelight.h | 38 ++++++++++------- src/RtspConnection.c | 54 +++++------------------- src/SdpGenerator.c | 10 ++++- src/VideoDepacketizer.c | 8 +++- src/VideoFormats.c | 91 +++++++++++++++++++++++++++++++++++++++++ src/VideoFormats.h | 25 +++++++++++ 6 files changed, 163 insertions(+), 63 deletions(-) create mode 100644 src/VideoFormats.c create mode 100644 src/VideoFormats.h diff --git a/src/Limelight.h b/src/Limelight.h index 552d54ad4..e47139051 100644 --- a/src/Limelight.h +++ b/src/Limelight.h @@ -222,23 +222,27 @@ typedef struct _DECODE_UNIT { // Passed in StreamConfiguration.supportedVideoFormats to specify supported codecs // and to DecoderRendererSetup() to specify selected codec. -#define VIDEO_FORMAT_H264 0x0001 // H.264 High Profile -#define VIDEO_FORMAT_H264_HIGH8_444 0x0004 // H.264 High 4:4:4 8-bit Profile -#define VIDEO_FORMAT_H265 0x0100 // HEVC Main Profile -#define VIDEO_FORMAT_H265_MAIN10 0x0200 // HEVC Main10 Profile -#define VIDEO_FORMAT_H265_REXT8_444 0x0400 // HEVC RExt 4:4:4 8-bit Profile -#define VIDEO_FORMAT_H265_REXT10_444 0x0800 // HEVC RExt 4:4:4 10-bit Profile -#define VIDEO_FORMAT_AV1_MAIN8 0x1000 // AV1 Main 8-bit profile -#define VIDEO_FORMAT_AV1_MAIN10 0x2000 // AV1 Main 10-bit profile -#define VIDEO_FORMAT_AV1_HIGH8_444 0x4000 // AV1 High 4:4:4 8-bit profile -#define VIDEO_FORMAT_AV1_HIGH10_444 0x8000 // AV1 High 4:4:4 10-bit profile +#define VIDEO_FORMAT_H264 0x00000001 // H.264 High Profile +#define VIDEO_FORMAT_H264_HIGH8_444 0x00000004 // H.264 High 4:4:4 8-bit Profile +#define VIDEO_FORMAT_H265 0x00000100 // HEVC Main Profile +#define VIDEO_FORMAT_H265_MAIN10 0x00000200 // HEVC Main10 Profile +#define VIDEO_FORMAT_H265_REXT8_444 0x00000400 // HEVC RExt 4:4:4 8-bit Profile +#define VIDEO_FORMAT_H265_REXT10_444 0x00000800 // HEVC RExt 4:4:4 10-bit Profile +#define VIDEO_FORMAT_AV1_MAIN8 0x00001000 // AV1 Main 8-bit profile +#define VIDEO_FORMAT_AV1_MAIN10 0x00002000 // AV1 Main 10-bit profile +#define VIDEO_FORMAT_AV1_HIGH8_444 0x00004000 // AV1 High 4:4:4 8-bit profile +#define VIDEO_FORMAT_AV1_HIGH10_444 0x00008000 // AV1 High 4:4:4 10-bit profile +#define VIDEO_FORMAT_MPEG2 0x00010000 // MPEG-2/H.262 video (Sunshine protocol extension) +#define VIDEO_FORMAT_H263P 0x00040000 // H.263+ video (Sunshine protocol extension) // Masks for clients to use to match video codecs without profile-specific details. -#define VIDEO_FORMAT_MASK_H264 0x000F -#define VIDEO_FORMAT_MASK_H265 0x0F00 -#define VIDEO_FORMAT_MASK_AV1 0xF000 -#define VIDEO_FORMAT_MASK_10BIT 0xAA00 -#define VIDEO_FORMAT_MASK_YUV444 0xCC04 +#define VIDEO_FORMAT_MASK_H264 0x0000000F +#define VIDEO_FORMAT_MASK_H265 0x00000F00 +#define VIDEO_FORMAT_MASK_AV1 0x0000F000 +#define VIDEO_FORMAT_MASK_MPEG2 0x00010000 +#define VIDEO_FORMAT_MASK_H263P 0x00040000 +#define VIDEO_FORMAT_MASK_10BIT 0x0000AA00 +#define VIDEO_FORMAT_MASK_YUV444 0x0000CC04 // If set in the renderer capabilities field, this flag will cause audio/video data to // be submitted directly from the receive thread. This should only be specified if the @@ -517,11 +521,15 @@ void LiInitializeConnectionCallbacks(PCONNECTION_LISTENER_CALLBACKS clCallbacks) #define SCM_HEVC_REXT10_444 0x00100000 // Sunshine extension #define SCM_AV1_HIGH8_444 0x00200000 // Sunshine extension #define SCM_AV1_HIGH10_444 0x00400000 // Sunshine extension +#define SCM_MPEG2 0x00800000 // Sunshine protocol extension +#define SCM_H263P 0x02000000 // Sunshine protocol extension // SCM masks to identify various codec capabilities #define SCM_MASK_H264 (SCM_H264 | SCM_H264_HIGH8_444) #define SCM_MASK_HEVC (SCM_HEVC | SCM_HEVC_MAIN10 | SCM_HEVC_REXT8_444 | SCM_HEVC_REXT10_444) #define SCM_MASK_AV1 (SCM_AV1_MAIN8 | SCM_AV1_MAIN10 | SCM_AV1_HIGH8_444 | SCM_AV1_HIGH10_444) +#define SCM_MASK_MPEG2 SCM_MPEG2 +#define SCM_MASK_H263P SCM_H263P #define SCM_MASK_10BIT (SCM_HEVC_MAIN10 | SCM_HEVC_REXT10_444 | SCM_AV1_MAIN10 | SCM_AV1_HIGH10_444) #define SCM_MASK_YUV444 (SCM_H264_HIGH8_444 | SCM_HEVC_REXT8_444 | SCM_HEVC_REXT10_444 | SCM_AV1_HIGH8_444 | SCM_AV1_HIGH10_444) diff --git a/src/RtspConnection.c b/src/RtspConnection.c index 33b1d431a..643ccc72a 100644 --- a/src/RtspConnection.c +++ b/src/RtspConnection.c @@ -1,5 +1,6 @@ #include "Limelight-internal.h" #include "Rtsp.h" +#include "VideoFormats.h" #define RTSP_CONNECT_TIMEOUT_SEC 10 #define RTSP_RECEIVE_TIMEOUT_SEC 15 @@ -1088,52 +1089,17 @@ int performRtspHandshake(PSERVER_INFORMATION serverInfo) { goto Exit; } - if ((StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_AV1) && strstr(response.payload, "AV1/90000")) { - if ((serverInfo->serverCodecModeSupport & SCM_AV1_HIGH10_444) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH10_444)) { - NegotiatedVideoFormat = VIDEO_FORMAT_AV1_HIGH10_444; - } - else if ((serverInfo->serverCodecModeSupport & SCM_AV1_MAIN10) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN10)) { - NegotiatedVideoFormat = VIDEO_FORMAT_AV1_MAIN10; - } - else if ((serverInfo->serverCodecModeSupport & SCM_AV1_HIGH8_444) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH8_444)) { - NegotiatedVideoFormat = VIDEO_FORMAT_AV1_HIGH8_444; - } - else { - NegotiatedVideoFormat = VIDEO_FORMAT_AV1_MAIN8; - } - } - else if ((StreamConfig.supportedVideoFormats & VIDEO_FORMAT_MASK_H265) && strstr(response.payload, "sprop-parameter-sets=AAAAAU")) { - // The RTSP DESCRIBE reply will contain a collection of SDP media attributes that - // describe the various supported video stream formats and include the SPS, PPS, - // and VPS (if applicable). We will use this information to determine whether the - // server can support HEVC. For some reason, they still set the MIME type of the HEVC - // format to H264, so we can't just look for the HEVC MIME type. What we'll do instead is - // look for the base 64 encoded VPS NALU prefix that is unique to the HEVC bitstream. - if ((serverInfo->serverCodecModeSupport & SCM_HEVC_REXT10_444) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_REXT10_444)) { - NegotiatedVideoFormat = VIDEO_FORMAT_H265_REXT10_444; - } - else if ((serverInfo->serverCodecModeSupport & SCM_HEVC_MAIN10) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_MAIN10)) { - NegotiatedVideoFormat = VIDEO_FORMAT_H265_MAIN10; - } - else if ((serverInfo->serverCodecModeSupport & SCM_HEVC_REXT8_444) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H265_REXT8_444)) { - NegotiatedVideoFormat = VIDEO_FORMAT_H265_REXT8_444; - } - else { - NegotiatedVideoFormat = VIDEO_FORMAT_H265; - } + NegotiatedVideoFormat = chooseVideoFormat(StreamConfig.supportedVideoFormats, + serverInfo->serverCodecModeSupport, response.payload); + if (NegotiatedVideoFormat == 0) { + Limelog("No mutually supported video codec found in RTSP DESCRIBE response\n"); + ret = -1; + goto Exit; } - else { - if ((serverInfo->serverCodecModeSupport & SCM_H264_HIGH8_444) && (StreamConfig.supportedVideoFormats & VIDEO_FORMAT_H264_HIGH8_444)) { - NegotiatedVideoFormat = VIDEO_FORMAT_H264_HIGH8_444; - } - else { - NegotiatedVideoFormat = VIDEO_FORMAT_H264; - } - // Dimensions over 4096 are only supported with HEVC on NVENC - if (StreamConfig.width > 4096 || StreamConfig.height > 4096) { - Limelog("WARNING: Host PC doesn't support HEVC. Streaming at resolutions above 4K using H.264 will likely fail!\n"); - } + if ((NegotiatedVideoFormat & VIDEO_FORMAT_MASK_H264) && + (StreamConfig.width > 4096 || StreamConfig.height > 4096)) { + Limelog("WARNING: Streaming at resolutions above 4K using H.264 will likely fail!\n"); } // Look for the SDP attribute that indicates we're dealing with a server that supports RFI diff --git a/src/SdpGenerator.c b/src/SdpGenerator.c index 5ea5cbbed..0ca207f15 100644 --- a/src/SdpGenerator.c +++ b/src/SdpGenerator.c @@ -1,3 +1,4 @@ +#include "VideoFormats.h" #include "Limelight-internal.h" #include @@ -438,8 +439,13 @@ static PSDP_OPTION getAttributesList(char*urlSafeAddr) { snprintf(payloadStr, sizeof(payloadStr), "%d", slicesPerFrame); err |= addAttributeString(&optionHead, "x-nv-video[0].videoEncoderSlicesPerFrame", payloadStr); - if (NegotiatedVideoFormat & VIDEO_FORMAT_MASK_AV1) { - err |= addAttributeString(&optionHead, "x-nv-vqos[0].bitStreamFormat", "2"); + if (NegotiatedVideoFormat & (VIDEO_FORMAT_MASK_AV1 | VIDEO_FORMAT_MASK_MPEG2 | VIDEO_FORMAT_MASK_H263P)) { + // Sunshine codec extensions use GameStream framing and shared wire identifiers. + snprintf(payloadStr, sizeof(payloadStr), "%d", getVideoBitStreamFormat(NegotiatedVideoFormat)); + if (NegotiatedVideoFormat & (VIDEO_FORMAT_MASK_MPEG2 | VIDEO_FORMAT_MASK_H263P)) { + err |= addAttributeString(&optionHead, "x-nv-clientSupportHevc", "0"); + } + err |= addAttributeString(&optionHead, "x-nv-vqos[0].bitStreamFormat", payloadStr); } else if (NegotiatedVideoFormat & VIDEO_FORMAT_MASK_H265) { err |= addAttributeString(&optionHead, "x-nv-clientSupportHevc", "1"); diff --git a/src/VideoDepacketizer.c b/src/VideoDepacketizer.c index 69ec1dbc2..5202cb532 100644 --- a/src/VideoDepacketizer.c +++ b/src/VideoDepacketizer.c @@ -84,6 +84,10 @@ typedef struct _LENTRY_INTERNAL { #define HEVC_NAL_TYPE_FILLER 38 #define HEVC_NAL_TYPE_SEI 39 +static bool isBitstreamPassthroughFormat(void) { + return NegotiatedVideoFormat & (VIDEO_FORMAT_MASK_AV1 | VIDEO_FORMAT_MASK_MPEG2 | VIDEO_FORMAT_MASK_H263P); +} + // Init void initializeVideoDepacketizer(int pktSize) { PltCreateMutex(&stateLock); @@ -248,8 +252,8 @@ void validateDecodeUnitForPlayback(PDECODE_UNIT decodeUnit) { LC_ASSERT_VT(decodeUnit->bufferList->next->next->bufferType == BUFFER_TYPE_PPS); LC_ASSERT_VT(decodeUnit->bufferList->next->next->next != NULL); } - else if (NegotiatedVideoFormat & VIDEO_FORMAT_MASK_AV1) { - // We don't parse the AV1 bitstream + else if (isBitstreamPassthroughFormat()) { + // We forward AV1, MPEG-2, and H.263+ bitstreams without NAL parsing. LC_ASSERT_VT(decodeUnit->bufferList->bufferType == BUFFER_TYPE_PICDATA); } else { diff --git a/src/VideoFormats.c b/src/VideoFormats.c new file mode 100644 index 000000000..60d542eb8 --- /dev/null +++ b/src/VideoFormats.c @@ -0,0 +1,91 @@ +/** + * @file VideoFormats.c + * @brief Negotiate modern and legacy video codecs using shared protocol capabilities. + */ +#include "VideoFormats.h" + +#include + +int chooseVideoFormat(int supportedVideoFormats, int serverCodecModeSupport, const char* sdpPayload) { + if (sdpPayload == NULL) { + return 0; + } + + if (strstr(sdpPayload, "AV1/90000") != NULL) { + if ((supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH10_444) && (serverCodecModeSupport & SCM_AV1_HIGH10_444)) { + return VIDEO_FORMAT_AV1_HIGH10_444; + } + if ((supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN10) && (serverCodecModeSupport & SCM_AV1_MAIN10)) { + return VIDEO_FORMAT_AV1_MAIN10; + } + if ((supportedVideoFormats & VIDEO_FORMAT_AV1_HIGH8_444) && (serverCodecModeSupport & SCM_AV1_HIGH8_444)) { + return VIDEO_FORMAT_AV1_HIGH8_444; + } + if ((supportedVideoFormats & VIDEO_FORMAT_AV1_MAIN8) && + (serverCodecModeSupport == 0 || (serverCodecModeSupport & SCM_AV1_MAIN8))) { + return VIDEO_FORMAT_AV1_MAIN8; + } + } + + // GameStream identifies HEVC support using the VPS prefix in this attribute. + if (strstr(sdpPayload, "sprop-parameter-sets=AAAAAU") != NULL) { + if ((supportedVideoFormats & VIDEO_FORMAT_H265_REXT10_444) && (serverCodecModeSupport & SCM_HEVC_REXT10_444)) { + return VIDEO_FORMAT_H265_REXT10_444; + } + if ((supportedVideoFormats & VIDEO_FORMAT_H265_MAIN10) && (serverCodecModeSupport & SCM_HEVC_MAIN10)) { + return VIDEO_FORMAT_H265_MAIN10; + } + if ((supportedVideoFormats & VIDEO_FORMAT_H265_REXT8_444) && (serverCodecModeSupport & SCM_HEVC_REXT8_444)) { + return VIDEO_FORMAT_H265_REXT8_444; + } + if ((supportedVideoFormats & VIDEO_FORMAT_H265) && + (serverCodecModeSupport == 0 || (serverCodecModeSupport & SCM_HEVC))) { + return VIDEO_FORMAT_H265; + } + } + + if ((supportedVideoFormats & VIDEO_FORMAT_H264_HIGH8_444) && (serverCodecModeSupport & SCM_H264_HIGH8_444)) { + return VIDEO_FORMAT_H264_HIGH8_444; + } + if ((supportedVideoFormats & VIDEO_FORMAT_H264) && + (serverCodecModeSupport == 0 || (serverCodecModeSupport & SCM_H264))) { + return VIDEO_FORMAT_H264; + } + + // Legacy codecs are opt-in Sunshine protocol extensions, available to any client. + if ((supportedVideoFormats & VIDEO_FORMAT_MPEG2) && (serverCodecModeSupport & SCM_MPEG2) && + (strstr(sdpPayload, "MPV/90000") != NULL || strstr(sdpPayload, "MPEG2/90000") != NULL || + strstr(sdpPayload, "MPEG-2/90000") != NULL)) { + return VIDEO_FORMAT_MPEG2; + } + if ((supportedVideoFormats & VIDEO_FORMAT_H263P) && (serverCodecModeSupport & SCM_H263P) && + (strstr(sdpPayload, "H263-1998/90000") != NULL || strstr(sdpPayload, "H263-2000/90000") != NULL)) { + return VIDEO_FORMAT_H263P; + } + + return 0; +} + +int getVideoBitStreamFormat(int videoFormat) { + switch (videoFormat) { + case VIDEO_FORMAT_H264: + case VIDEO_FORMAT_H264_HIGH8_444: + return 0; + case VIDEO_FORMAT_H265: + case VIDEO_FORMAT_H265_MAIN10: + case VIDEO_FORMAT_H265_REXT8_444: + case VIDEO_FORMAT_H265_REXT10_444: + return 1; + case VIDEO_FORMAT_AV1_MAIN8: + case VIDEO_FORMAT_AV1_MAIN10: + case VIDEO_FORMAT_AV1_HIGH8_444: + case VIDEO_FORMAT_AV1_HIGH10_444: + return 2; + case VIDEO_FORMAT_MPEG2: + return 3; + case VIDEO_FORMAT_H263P: + return 4; + default: + return -1; + } +} diff --git a/src/VideoFormats.h b/src/VideoFormats.h new file mode 100644 index 000000000..d85e2e8ef --- /dev/null +++ b/src/VideoFormats.h @@ -0,0 +1,25 @@ +/** + * @file VideoFormats.h + * @brief Internal codec negotiation and GameStream wire-format helpers. + */ +#pragma once + +#include "Limelight.h" + +/** + * @brief Choose a mutually supported codec without giving legacy codecs precedence. + * + * @param supportedVideoFormats Client decoder capability mask. + * @param serverCodecModeSupport Host codec capability mask; zero permits baseline profiles on older hosts. + * @param sdpPayload RTSP DESCRIBE payload advertising the host codecs. + * @return One VIDEO_FORMAT value, or zero when no supported codec is available. + */ +int chooseVideoFormat(int supportedVideoFormats, int serverCodecModeSupport, const char* sdpPayload); + +/** + * @brief Map a negotiated codec to its GameStream bitStreamFormat value. + * + * @param videoFormat One negotiated VIDEO_FORMAT value. + * @return Wire value from zero through four, or -1 for an invalid format. + */ +int getVideoBitStreamFormat(int videoFormat);