Implement SEI messages and frontend config to inject Alpha metadata in the stream

bin/release-static/sequence2K.cfg

@@ -0,0 +1,13 @@
+# gst-launch-1.0 videotestsrc num-buffers=60 pattern=ball motion=sweep ! "video/x-raw,width=1920,height=1080,framerate=60/1" ! videoconvert ! video/x-raw,format=I420 ! filesink location=./output2.yuv
+#======== File I/O ===============
+InputFile                     : output2.yuv   # Input file
+InputBitDepth                 : 8           # Input bitdepth
+InputChromaFormat             : 420         # Ratio of luminance to chrominance samples
+FrameRate                     : 60          # Frame Rate per second (numerator)
+FrameScale                    : 1           # Frame Scale per second (denominator)
+FrameSkip                     : 0           # Number of frames to be skipped in input
+SourceWidth                   : 1920        # Input frame width
+SourceHeight                  : 1080        # Input frame height
+FramesToBeEncoded             : 5         # Number of frames to be coded
+
+Level                         : 4.1

bin/release-static/sequenceAlpha.cfg

@@ -0,0 +1,13 @@
+# generated with: gst-launch-1.0 videotestsrc num-buffers=60 pattern=ball ! "video/x-raw,width=1920,height=1080,framerate=60/1" ! videoconvert ! video/x-raw,format=I420 ! filesink location=./alpha_8.yuv
+#======== File I/O ===============
+InputFile                     : alpha_8.yuv   # Input file
+InputBitDepth                 : 8           # Input bitdepth
+InputChromaFormat             : 420         # Ratio of luminance to chrominance samples
+FrameRate                     : 60          # Frame Rate per second (numerator)
+FrameScale                    : 1           # Frame Scale per second (denominator)
+FrameSkip                     : 0           # Number of frames to be skipped in input
+SourceWidth                   : 1920        # Input frame width
+SourceHeight                  : 1080        # Input frame height
+FramesToBeEncoded             : 5         # Number of frames to be coded
+
+Level                         : 4.1

bin/release-static/sequenceGray.cfg

@@ -0,0 +1,12 @@
+#======== File I/O ===============
+InputFile                     : gray_8.yuv   # Input file
+InputBitDepth                 : 8           # Input bitdepth
+InputChromaFormat             : 400         # Ratio of luminance to chrominance samples
+FrameRate                     : 60          # Frame Rate per second (numerator)
+FrameScale                    : 1           # Frame Scale per second (denominator)
+FrameSkip                     : 0           # Number of frames to be skipped in input
+SourceWidth                   : 1920        # Input frame width
+SourceHeight                  : 1080        # Input frame height
+FramesToBeEncoded             : 60         # Number of frames to be coded
+
+Level                         : 4.1

cfg/randomaccess_fast.cfg

@@ -42,7 +42,7 @@ LoopFilterCrBetaOffset_div2   : 0           # base_param: -12 ~ 12
 LoopFilterCrTcOffset_div2     : 0           # base_param: -12 ~ 12
 
 #=========== Misc. ============
-InternalBitDepth              : 10          # codec operating bit-depth
+InternalBitDepth              : 8          # codec operating bit-depth
 SEIDecodedPictureHash         : 0
 CbQpOffset                    : 0
 CrQpOffset                    : 0
@@ -66,7 +66,7 @@ MaxNumMergeCand               : 6      # Maximum number of merge candidates
 Affine                        : 5      # Affine prediction: 0: disabled, 1: vtm, 2-5: fast modes
 ALF                           : 1      # Adpative Loop Filter: 0: disabled, 1: enabled
 ALFSpeed                      : 1      # ALF speed (skip filtering of non-referenced frames) [0-1]
-ALFUnitSize                   : 128    # ALF processing size 
+ALFUnitSize                   : 128    # ALF processing size
 AllowDisFracMMVD              : 1      # Disable fractional MVD in MMVD mode adaptively
 BCW                           : 0      # Enable Generalized Bi-prediction(Bcw) 0: disabled, 1: enabled, 2: fast
 BIM                           : 1      # Block importance mapping (basic temporal RDO based on MCTF).
@@ -104,7 +104,7 @@ TMVPMode                      : 1      # TMVP mode 0: TMVP disable for all slice
 UseNonLinearAlfChroma         : 0      # Non-linear adaptive loop filters for Chroma Channels: 0: disabled, 1: enabled
 UseNonLinearAlfLuma           : 0      # Non-linear adaptive loop filters for Luma Channel: 0: disabled, 1: enabled
 
-# SCC configuration           
+# SCC configuration
 FastSearchSCC                 : 2      # Search mode for SCC (0:use non SCC-search 1:Selective 2:DiamondSCC 3:FastDiamondSCC)
 BDPCM                         : 2      # BDPCM (0: off, 1: luma and chroma, 2: BDPCM with SCC detection)
 IBC                           : 2      # IBC mode: 0: off, 1: IBC, 2: IBC with SCC detection
@@ -113,7 +113,7 @@ TransformSkip                 : 2      # Transform skipping, 0: disabled, 1: ena
 ChromaTS                      : 0      # Chroma transform skip: 0: disabled, 1: enabled
 TransformSkipLog2MaxSize      : 4
 
-# Fast options                
+# Fast options
 QtbttExtraFast                : 3      # Non-VTM compatible QTBTT speed-ups: 0: disabled, 1: enabled
 ContentBasedFastQtbt          : 1      # Signal based QTBT speed-up: 0: disabled, 1: enabled
 PBIntraFast                   : 1      # Intra mode pre-check dependent on best Inter mode, skip intra if it is not probable (0:off ... 2:fastest)

cfg/two_layers_gray.cfg

@@ -0,0 +1,19 @@
+#======== Layers ===============
+MaxLayers                     : 2
+MaxSublayers                  : 7
+AllIndependentLayersFlag      : 1
+#======== Alpha ================
+SEIAlphaChannel               : 1
+SEIScalabityDimension         : 1
+#======== OLSs =================
+EachLayerIsAnOlsFlag          : 1
+NumOutputLayerSets            : 2
+NumPTLsInVPS                  : 1
+#======== Layer-0 ===============
+LayerId0                      : 0
+#======== Layer-1 ===============
+LayerId1                      : 1
+#======== OLS-0 ===============
+OlsPTLIdx0                    : 0
+#======== OLS-1 ===============
+OlsPTLIdx1                    : 1

include/vvenc/vvencCfg.h

@@ -557,6 +557,10 @@ typedef struct vvenc_config
   bool                m_bufferingPeriodSEIEnabled;
   bool                m_pictureTimingSEIEnabled;
   bool                m_decodingUnitInfoSEIEnabled;
+#if ENABLE_SPATIAL_SCALABLE
+  bool                m_alphaChannelInfoSEIEnabled;
+  bool                m_scalabilityDimensionInfoSEIEnabled;
+#endif
 
   int8_t              m_entropyCodingSyncEnabled;
   bool                m_entryPointsPresent;

source/Lib/CommonLib/SEI.cpp

@@ -139,6 +139,8 @@ const char *SEI::getSEIMessageString(SEI::PayloadType payloadType)
     case SEI::GENERALIZED_CUBEMAP_PROJECTION:       return "Generalized cubemap projection";
     case SEI::SAMPLE_ASPECT_RATIO_INFO:             return "Sample aspect ratio information";
     case SEI::SUBPICTURE_LEVEL_INFO:                return "Subpicture level information";
+    case SEI::ALPHA_CHANNEL:                        return "Alpha channel information";
+    case SEI::SCALABILITY_DIMENSION_INFO:           return "Scalability dimension information";
     default:                                        return "Unknown";
   }
 }

source/Lib/CommonLib/SEI.h

@@ -85,6 +85,8 @@ class SEI
     ALTERNATIVE_TRANSFER_CHARACTERISTICS = 147,
     AMBIENT_VIEWING_ENVIRONMENT          = 148,
     CONTENT_COLOUR_VOLUME                = 149,
+    ALPHA_CHANNEL                        = 165,
+    SCALABILITY_DIMENSION_INFO           = 208,
   };
 
   SEI() {}
@@ -229,6 +231,57 @@ class SEISampleAspectRatioInfo : public SEI
   int                   sariSarHeight;
 };
 
+class SEIAlphaChannelInfo : public SEI
+{
+public:
+  PayloadType payloadType() const { return ALPHA_CHANNEL; }
+
+  SEIAlphaChannelInfo()
+    : alphaChannelCancelFlag(false)
+    , alphaChannelUseIdc(0)
+    , alphaChannelBitDepthMinus8(0)
+    , alphaTransparentValue(0)
+    , alphaOpaqueValue(0)
+    , alphaChannelIncrFlag(false)
+    , alphaChannelClipFlag(false)
+    , alphaChannelClipTypeFlag(false)
+    {}
+  virtual ~SEIAlphaChannelInfo() {}
+
+  bool     alphaChannelCancelFlag;
+  uint32_t alphaChannelUseIdc;
+  uint32_t alphaChannelBitDepthMinus8;
+  uint32_t alphaTransparentValue;
+  uint32_t alphaOpaqueValue;
+  bool     alphaChannelIncrFlag;
+  bool     alphaChannelClipFlag;
+  bool     alphaChannelClipTypeFlag;
+};
+
+class SEIScalabilityDimensionInfo : public SEI
+{
+public:
+  PayloadType payloadType() const { return SCALABILITY_DIMENSION_INFO; }
+
+  SEIScalabilityDimensionInfo()
+    : sdiMaxLayersMinus1(0)
+    , sdiMultiviewInfoFlag(false)
+    , sdiAuxiliaryInfoFlag(false)
+    , sdiViewIdLenMinus1(0)
+    {}
+  virtual ~SEIScalabilityDimensionInfo() {}
+
+  uint32_t sdiMaxLayersMinus1;
+  bool sdiMultiviewInfoFlag;
+  bool sdiAuxiliaryInfoFlag;
+  u_int32_t sdiViewIdLenMinus1;
+  std::vector<u_int32_t> sdiLayerId;
+  std::vector<u_int32_t> sdiViewIdVal;
+  std::vector<u_int32_t> sdiAuxId;
+  std::vector<u_int32_t> sdiNumAssociatedPrimaryLayersMinus1;
+  std::vector<std::vector<u_int32_t>>  sdiAssociatedPrimaryLayerIdx;
+};
+
 static const uint32_t ISO_IEC_11578_LEN=16;
 
 class SEIuserDataUnregistered : public SEI

source/Lib/EncoderLib/EncGOP.cpp

@@ -3080,6 +3080,11 @@ void EncGOP::xWriteLeadingSEIs( const Picture& pic, AccessUnitList& accessUnit )
     bpPresentInAU = true;
   }
 
+  if (m_pcEncCfg->m_scalabilityDimensionInfoSEIEnabled && m_pcEncCfg->m_alphaChannelInfoSEIEnabled)
+  {
+    m_seiEncoder.initAlphaChannelSEI( leadingSeiMessages, slice );
+  }
+
 //  if (m_pcEncCfg->m_dependentRAPIndicationSEIEnabled && slice->isDRAP )
 //  {
 //    SEIDependentRAPIndication *dependentRAPIndicationSEI = new SEIDependentRAPIndication();

source/Lib/EncoderLib/SEIEncoder.cpp

@@ -233,6 +233,50 @@ void SEIEncoder::initDecodedPictureHashSEI( SEIDecodedPictureHash& dphSei, const
   }
 }
 
+void SEIEncoder::initAlphaChannelSEI( SEIMessages& seiMessages, const Slice *slice )
+{
+#if ENABLE_SPATIAL_SCALABLE
+  // Alpha channel signalling depends on both SEI messages: sdi and aci
+  if (!(m_pcEncCfg->m_scalabilityDimensionInfoSEIEnabled && m_pcEncCfg->m_alphaChannelInfoSEIEnabled))
+  {
+    return;
+  }
+
+  CHECK( m_pcEncCfg->m_maxLayers != 2, "Writing Alpha Channel should have two layers" );
+  SEIScalabilityDimensionInfo *scalabityDimensionInfo = new SEIScalabilityDimensionInfo();
+  scalabityDimensionInfo->sdiMaxLayersMinus1 = slice->vps->maxLayers - 1;
+  scalabityDimensionInfo->sdiMultiviewInfoFlag = 0;
+  scalabityDimensionInfo->sdiAuxiliaryInfoFlag = 1;
+
+  scalabityDimensionInfo->sdiLayerId.resize(scalabityDimensionInfo->sdiMaxLayersMinus1+1);
+  scalabityDimensionInfo->sdiViewIdVal.resize(scalabityDimensionInfo->sdiMaxLayersMinus1+1);
+  scalabityDimensionInfo->sdiAuxId.resize(scalabityDimensionInfo->sdiMaxLayersMinus1+1);
+  scalabityDimensionInfo->sdiNumAssociatedPrimaryLayersMinus1.resize(scalabityDimensionInfo->sdiMaxLayersMinus1+1);
+  scalabityDimensionInfo->sdiAssociatedPrimaryLayerIdx.resize(scalabityDimensionInfo->sdiMaxLayersMinus1+1);
+
+  scalabityDimensionInfo->sdiLayerId[0] = 0;  /* Layer 1 - 1 */
+  scalabityDimensionInfo->sdiAuxId[0] = 0;    /* Nothing */
+
+  scalabityDimensionInfo->sdiLayerId[1] = 1;  /* Layer 1 - 1 */
+  scalabityDimensionInfo->sdiAuxId[1] = 1;    /* Alpha */
+  scalabityDimensionInfo->sdiNumAssociatedPrimaryLayersMinus1[1] = 0;
+  std::vector<u_int32_t> ref_layer = {0};
+  scalabityDimensionInfo->sdiAssociatedPrimaryLayerIdx[1] = ref_layer;
+  seiMessages.push_back(scalabityDimensionInfo);
+
+  SEIAlphaChannelInfo *alphaInfoSEI = new SEIAlphaChannelInfo();
+  alphaInfoSEI->alphaChannelCancelFlag = false;
+  alphaInfoSEI->alphaChannelUseIdc = 1;
+  alphaInfoSEI->alphaChannelBitDepthMinus8 = slice->sps->bitDepths[CH_L] - 8;
+  alphaInfoSEI->alphaTransparentValue = 0;
+  alphaInfoSEI->alphaOpaqueValue = slice->sps->bitDepths[CH_L] == 8?255:1024;
+  alphaInfoSEI->alphaChannelIncrFlag = 0;
+  alphaInfoSEI->alphaChannelClipFlag = 0;
+  alphaInfoSEI->alphaChannelClipTypeFlag = 0;
+  seiMessages.push_back(alphaInfoSEI);
+#endif
+}
+
 void SEIEncoder::initPictureTimingSEI( SEIMessages& seiMessages, SEIMessages& nestedSeiMessages, SEIMessages& duInfoSeiMessages, const Slice *slice, const uint32_t numDU, const bool bpPresentInAU)
 {
   // Picture timing depends on buffering period. When either of those is not disabled,

source/Lib/EncoderLib/SEIEncoder.h

@@ -81,6 +81,7 @@ class SEIEncoder
   void initBufferingPeriodSEI     ( SEIBufferingPeriod& bpSei, bool noLeadingPictures);
   void initPictureTimingSEI       ( SEIMessages& seiMessages, SEIMessages& nestedSeiMessages, SEIMessages& duInfoSeiMessages, const Slice *slice, const uint32_t numDU, const bool bpPresentInAU);
   void initDrapSEI                ( SEIDependentRAPIndication& drapSei) {};
+  void initAlphaChannelSEI        ( SEIMessages& seiMessages, const Slice *slice );
 
   void initSEIAlternativeTransferCharacteristics(SEIAlternativeTransferCharacteristics *seiAltTransCharacteristics);
   void initSEIMasteringDisplayColourVolume(SEIMasteringDisplayColourVolume *seiMDCV);

source/Lib/EncoderLib/SEIwrite.cpp

@@ -137,6 +137,12 @@ void SEIWriter::xWriteSEIpayloadData(OutputBitstream &bs, const SEI& sei, HRD &h
   case SEI::SAMPLE_ASPECT_RATIO_INFO:
     xWriteSEISampleAspectRatioInfo(*static_cast<const SEISampleAspectRatioInfo*>(&sei));
     break;
+  case SEI::ALPHA_CHANNEL:
+    xWriteSEIAlphaChannelInfo(*static_cast<const SEIAlphaChannelInfo*>(&sei));
+    break;
+  case SEI::SCALABILITY_DIMENSION_INFO:
+    xWriteSEIScalabilityDimensionInfo(*static_cast<const SEIScalabilityDimensionInfo*>(&sei));
+    break;
   default:
     THROW("Trying to write unhandled SEI message");
     break;
@@ -806,6 +812,58 @@ void SEIWriter::xWriteSEISampleAspectRatioInfo(const SEISampleAspectRatioInfo &s
   }
 }
 
+void SEIWriter::xWriteSEIAlphaChannelInfo(const SEIAlphaChannelInfo &sei)
+{
+  WRITE_FLAG( sei.alphaChannelCancelFlag,                                       "alpha_channel_cancel_flag" );
+  if(!sei.alphaChannelCancelFlag)
+  {
+    WRITE_CODE( sei.alphaChannelUseIdc, 3,                                      "alpha_channel_use_idc" );
+    WRITE_CODE( sei.alphaChannelBitDepthMinus8, 3,                              "alpha_channel_bit_depth_minus8" );
+    WRITE_CODE( sei.alphaTransparentValue, sei.alphaChannelBitDepthMinus8 + 9,  "alpha_channel_transparent_value" );
+    WRITE_CODE( sei.alphaOpaqueValue,sei.alphaChannelBitDepthMinus8 + 9,        "alpha_channel_opaque_value" );
+    WRITE_FLAG( sei.alphaChannelIncrFlag,                                       "alpha_channel_incr_flag" );
+    WRITE_FLAG( sei.alphaChannelClipFlag,                                       "alpha_channel_clip_flag" );
+    if ( sei.alphaChannelIncrFlag)
+    {
+      WRITE_FLAG( sei.alphaChannelClipTypeFlag,                                 "alpha_channel_clip_type_flag" );
+    }
+  }
+}
+
+void SEIWriter::xWriteSEIScalabilityDimensionInfo(const SEIScalabilityDimensionInfo &sei)
+{
+  WRITE_CODE( sei.sdiMaxLayersMinus1, 6,                                "sdi_max_layers_minus1");
+  WRITE_FLAG( sei.sdiMultiviewInfoFlag,                                 "sdi_multiview_info_flag");
+  WRITE_FLAG( sei.sdiAuxiliaryInfoFlag,                                 "sdi_auxiliary_info_flag");
+  if (sei.sdiMultiviewInfoFlag || sei.sdiAuxiliaryInfoFlag)
+  {
+    if (sei.sdiMultiviewInfoFlag)
+    {
+      WRITE_CODE( sei.sdiViewIdLenMinus1, 4,                            "sdi_view_id_len_minus1");
+    }
+    for (unsigned int i=0; i<=sei.sdiMaxLayersMinus1; i++)
+    {
+      WRITE_CODE( sei.sdiLayerId[i], 6,                                 "sdi_layer_id");
+      if (sei.sdiMultiviewInfoFlag)
+      {
+        WRITE_CODE( sei.sdiViewIdVal[i], sei.sdiViewIdLenMinus1+1,      "sdi_view_id_val");
+      }
+      if (sei.sdiAuxiliaryInfoFlag)
+      {
+        WRITE_CODE( sei.sdiAuxId[i], 8,                                 "sdi_aux_id");
+        if (sei.sdiAuxId[i]>0)
+        {
+          WRITE_CODE( sei.sdiNumAssociatedPrimaryLayersMinus1[i], 6,    "sdi_num_associated_primary_layers_minus1");
+          for (unsigned int j=0; j<=sei.sdiNumAssociatedPrimaryLayersMinus1[i]; j++)
+          {
+            WRITE_CODE( sei.sdiAssociatedPrimaryLayerIdx[i][j], 6,      "sdi_associated_primary_layer_idx");
+          }
+        }
+      }
+    }
+  }
+}
+
 void SEIWriter::xWriteSEIUserDataRegistered(const SEIUserDataRegistered &sei)
 {
   WRITE_CODE((sei.ituCountryCode>255) ? 0xff : sei.ituCountryCode, 8, "itu_t_t35_country_code");

source/Lib/EncoderLib/SEIwrite.h

@@ -82,6 +82,8 @@ class SEIWriter : public VLCWriter
   void xWriteSEIGeneralizedCubemapProjection      (const SEIGeneralizedCubemapProjection &sei);
   void xWriteSEISubpictureLevelInfo               (const SEISubpicureLevelInfo &sei);
   void xWriteSEISampleAspectRatioInfo             (const SEISampleAspectRatioInfo &sei);
+  void xWriteSEIAlphaChannelInfo                  (const SEIAlphaChannelInfo &sei);
+  void xWriteSEIScalabilityDimensionInfo          (const SEIScalabilityDimensionInfo &sei);
 
   void xWriteSEIUserDataRegistered(const SEIUserDataRegistered& sei);
   void xWriteSEIFilmGrainCharacteristics(const SEIFilmGrainCharacteristics& sei);

source/Lib/EncoderLib/VLCWriter.cpp

@@ -2242,7 +2242,7 @@ void  HLSWriter::codeProfileTierLevel    ( const ProfileTierLevel* ptl, bool pro
 
   WRITE_CODE( (uint32_t)ptl->levelIdc, 8 ,            "general_level_idc");
   WRITE_FLAG( ptl->frameOnlyConstraintFlag,           "ptl_frame_only_constraint_flag" );
-  WRITE_FLAG( ptl->multiLayerEnabledFlag,             "ptl_multilayer_enabled_flag"    );
+  WRITE_FLAG( 0,             "ptl_multilayer_enabled_flag"    );
   if(profileTierPresent)
   {
     codeConstraintInfo( &ptl->constraintInfo );

source/Lib/apputils/VVEncAppCfg.h

@@ -1015,6 +1015,10 @@ int parse( int argc, char* argv[], vvenc_config* c, std::ostream& rcOstr )
     ("SEIBufferingPeriod",                              c->m_bufferingPeriodSEIEnabled,                      "Control generation of buffering period SEI messages")
     ("SEIPictureTiming",                                c->m_pictureTimingSEIEnabled,                        "Control generation of picture timing SEI messages")
     ("SEIDecodingUnitInfo",                             c->m_decodingUnitInfoSEIEnabled,                     "Control generation of decoding unit information SEI message.")
+#if ENABLE_SPATIAL_SCALABLE
+    ("SEIAlphaChannel",                                 c->m_alphaChannelInfoSEIEnabled,                     "Control generation of alpha channel SEI message.")
+    ("SEIScalabityDimension",                           c->m_scalabilityDimensionInfoSEIEnabled,             "Control generation of scalability dimension SEI message.")
+#endif
     ("EnableDecodingParameterSet",                      c->m_decodingParameterSetEnabled,                    "Enable writing of Decoding Parameter Set")
     ("AccessUnitDelimiter,-aud",                        toAud,                                               "Enable Access Unit Delimiter NALUs, (default: auto - enable only if needed by dependent options)" , true)
     ("VuiParametersPresent,-vui",                       toVui,                                               "Enable generation of vui_parameters(), (default: auto - enable only if needed by dependent options)", true)

source/Lib/vvenc/vvencCfg.cpp

@@ -914,6 +914,8 @@ VVENC_DECL bool vvenc_init_config_parameter( vvenc_config *c )
         c->m_profile=vvencProfile::VVENC_MAIN_10;
       }
 #endif
+      std::cout << "Profile is " << c->m_profile << std::endl;
+      c->m_profile = vvencProfile::VVENC_MAIN_10;
     }
     else if (c->m_internChromaFormat==vvencChromaFormat::VVENC_CHROMA_422 || c->m_internChromaFormat==vvencChromaFormat::VVENC_CHROMA_444)
     {