vaapi: support VAProfileH264High10

see intel/libva#664

Signed-off-by: jianfeng.zheng <[email protected]>

Author: jianfeng.zheng

libavcodec/h264_slice.c

@@ -809,8 +809,15 @@ static enum AVPixelFormat get_pixel_format(H264Context *h, int force_callback)
                 *fmt++ = AV_PIX_FMT_YUV444P10;
         } else if (CHROMA422(h))
             *fmt++ = AV_PIX_FMT_YUV422P10;
-        else
+        else {
+#if CONFIG_H264_VAAPI_HWACCEL
+            // Just add as candidate. Whether VAProfileH264High10 usable or
+            // not is decided by vaapi_decode_make_config() defined in FFmpeg
+            // and vaQueryCodingProfile() defined in libva.
+            *fmt++ = AV_PIX_FMT_VAAPI;
+#endif
             *fmt++ = AV_PIX_FMT_YUV420P10;
+        }
         break;
     case 12:
         if (CHROMA444(h)) {

libavcodec/vaapi_decode.c

@@ -398,6 +398,11 @@ static const struct {
     MAP(MPEG4,       MPEG4_ADVANCED_SIMPLE,
                                MPEG4AdvancedSimple),
     MAP(MPEG4,       MPEG4_MAIN,      MPEG4Main   ),
+#if VA_CHECK_VERSION(1, 18, 0)
+    MAP(H264,        H264_HIGH_10_INTRA,
+                                      H264High10  ),
+    MAP(H264,        H264_HIGH_10,    H264High10  ),
+#endif
     MAP(H264,        H264_CONSTRAINED_BASELINE,
                            H264ConstrainedBaseline),
     MAP(H264,        H264_MAIN,       H264Main    ),

libavcodec/vaapi_encode_h264.c

@@ -23,6 +23,7 @@
 
 #include "libavutil/avassert.h"
 #include "libavutil/common.h"
+#include "libavutil/pixdesc.h"
 #include "libavutil/internal.h"
 #include "libavutil/opt.h"
 
@@ -301,10 +302,21 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
     H264RawPPS                        *pps = &priv->raw_pps;
     VAEncSequenceParameterBufferH264 *vseq = ctx->codec_sequence_params;
     VAEncPictureParameterBufferH264  *vpic = ctx->codec_picture_params;
+    const AVPixFmtDescriptor *desc;
+    int bit_depth;
 
     memset(sps, 0, sizeof(*sps));
     memset(pps, 0, sizeof(*pps));
 
+    desc = av_pix_fmt_desc_get(priv->common.input_frames->sw_format);
+    av_assert0(desc);
+    if (desc->nb_components == 1 || desc->log2_chroma_w != 1 || desc->log2_chroma_h != 1) {
+        av_log(avctx, AV_LOG_ERROR, "Chroma format of input pixel format "
+                "%s is not supported.\n", desc->name);
+        return AVERROR(EINVAL);
+    }
+    bit_depth = desc->comp[0].depth;
+
     sps->nal_unit_header.nal_ref_idc   = 3;
     sps->nal_unit_header.nal_unit_type = H264_NAL_SPS;
 
@@ -314,11 +326,11 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
         avctx->profile == FF_PROFILE_H264_MAIN)
         sps->constraint_set1_flag = 1;
 
-    if (avctx->profile == FF_PROFILE_H264_HIGH)
+    if (avctx->profile == FF_PROFILE_H264_HIGH || avctx->profile == FF_PROFILE_H264_HIGH_10)
         sps->constraint_set3_flag = ctx->gop_size == 1;
 
     if (avctx->profile == FF_PROFILE_H264_MAIN ||
-        avctx->profile == FF_PROFILE_H264_HIGH) {
+        avctx->profile == FF_PROFILE_H264_HIGH || avctx->profile == FF_PROFILE_H264_HIGH_10) {
         sps->constraint_set4_flag = 1;
         sps->constraint_set5_flag = ctx->b_per_p == 0;
     }
@@ -359,6 +371,8 @@ static int vaapi_encode_h264_init_sequence_params(AVCodecContext *avctx)
 
     sps->seq_parameter_set_id = 0;
     sps->chroma_format_idc    = 1;
+    sps->bit_depth_luma_minus8 = bit_depth - 8;
+    sps->bit_depth_chroma_minus8 = bit_depth - 8;
 
     sps->log2_max_frame_num_minus4 = 4;
     sps->pic_order_cnt_type        = ctx->max_b_depth ? 0 : 2;
@@ -1144,6 +1158,9 @@ static av_cold int vaapi_encode_h264_configure(AVCodecContext *avctx)
 }
 
 static const VAAPIEncodeProfile vaapi_encode_h264_profiles[] = {
+#if VA_CHECK_VERSION(1, 18, 0)
+    { FF_PROFILE_H264_HIGH_10, 10, 3, 1, 1, VAProfileH264High10 },
+#endif
     { FF_PROFILE_H264_HIGH, 8, 3, 1, 1, VAProfileH264High },
     { FF_PROFILE_H264_MAIN, 8, 3, 1, 1, VAProfileH264Main },
     { FF_PROFILE_H264_CONSTRAINED_BASELINE,
@@ -1208,10 +1225,9 @@ static av_cold int vaapi_encode_h264_init(AVCodecContext *avctx)
         av_log(avctx, AV_LOG_ERROR, "H.264 extended profile "
                "is not supported.\n");
         return AVERROR_PATCHWELCOME;
-    case FF_PROFILE_H264_HIGH_10:
     case FF_PROFILE_H264_HIGH_10_INTRA:
-        av_log(avctx, AV_LOG_ERROR, "H.264 10-bit profiles "
-               "are not supported.\n");
+        av_log(avctx, AV_LOG_ERROR, "H.264 high 10 intra profile "
+               "is not supported.\n");
         return AVERROR_PATCHWELCOME;
     case FF_PROFILE_H264_HIGH_422:
     case FF_PROFILE_H264_HIGH_422_INTRA:
@@ -1304,6 +1320,7 @@ static const AVOption vaapi_encode_h264_options[] = {
     { PROFILE("constrained_baseline", FF_PROFILE_H264_CONSTRAINED_BASELINE) },
     { PROFILE("main",                 FF_PROFILE_H264_MAIN) },
     { PROFILE("high",                 FF_PROFILE_H264_HIGH) },
+    { PROFILE("high10",               FF_PROFILE_H264_HIGH_10) },
 #undef PROFILE
 
     { "level", "Set level (level_idc)",