[FFmpeg-devel] [PATCH v1 2/3] libavcodec: Add thumbnail output to vaapi_h264 decoder
Zhou, Zachary
zachary.zhou at intel.com
Mon Apr 8 10:47:58 EEST 2019
> -----Original Message-----
> From: Zhou, Zachary
> Sent: Monday, April 8, 2019 3:10 PM
> To: ffmpeg-devel at ffmpeg.org
> Cc: Zhou, Zachary <zachary.zhou at intel.com>
> Subject: [PATCH v1 2/3] libavcodec: Add thumbnail output to vaapi_h264
> decoder
>
> This is sample code for reference
>
> HW support for decode+scaling in a single HW command (VDBOX+SFC).
> The primary target usage is video analytics, but can be used playback,
> transcoding, etc.
>
> For VAAPI -
> https://github.com/intel/libva
> basically, it allows multiple outputs (in different resolutions) using the decode
> context in a single call (you can search for “additional_outputs” in va.h).
>
> VAAPI sample code -
> https://github.com/intel/libva-
> utils/commit/957a269f02b00760b7e807643c821ee26abc529b
> ---
> libavcodec/avcodec.h | 8 +++
> libavcodec/decode.c | 16 +++++
> libavcodec/options_table.h | 4 ++
> libavcodec/vaapi_decode.c | 122 ++++++++++++++++++++++++++++++++++---
> libavcodec/vaapi_decode.h | 30 +++++++++
> libavcodec/vaapi_h264.c | 13 ++++
> 6 files changed, 185 insertions(+), 8 deletions(-)
>
> diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h index
> 0ce22ec4fa..86ab18011c 100644
> --- a/libavcodec/avcodec.h
> +++ b/libavcodec/avcodec.h
> @@ -3357,6 +3357,14 @@ typedef struct AVCodecContext {
> * - encoding: unused
> */
> int discard_damaged_percentage;
> +
> + /*
> + * VDSFC options
> + */
> + int sfc_flags;
> + int sfc_format;
> + int sfc_width;
> + int sfc_height;
> } AVCodecContext;
>
> #if FF_API_CODEC_GET_SET
> diff --git a/libavcodec/decode.c b/libavcodec/decode.c index
> a32ff2fcd3..df5f5b6eed 100644
> --- a/libavcodec/decode.c
> +++ b/libavcodec/decode.c
> @@ -1795,6 +1795,22 @@ FF_ENABLE_DEPRECATION_WARNINGS
> frame->channels = avctx->channels;
> break;
> }
> +
> + //sfc side data
> + if (avctx->sfc_flags) {
> + AVFrameSideData *frame_sfc_sd = av_frame_new_side_data(frame,
> + AV_FRAME_DATA_SFC_INFO,
> + sizeof(AVSFCInfo));
> + if (frame_sfc_sd) {
> + av_dict_set_int(&frame_sfc_sd->metadata, "sfc_flags", avctx-
> >sfc_flags, 0);
> + av_dict_set_int(&frame_sfc_sd->metadata, "sfc_width", avctx-
> >sfc_width, 0);
> + av_dict_set_int(&frame_sfc_sd->metadata, "sfc_height", avctx-
> >sfc_height, 0);
> + av_dict_set_int(&frame_sfc_sd->metadata, "sfc_format",
> + avctx->sfc_format, 0);
> +
> + av_log(avctx, AV_LOG_DEBUG, "VDSFC new side data\n");
> + }
> + }
> +
> return 0;
> }
>
> diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h index
> a3235bcd57..c14ff08678 100644
> --- a/libavcodec/options_table.h
> +++ b/libavcodec/options_table.h
> @@ -480,6 +480,10 @@ static const AVOption avcodec_options[] =
> { {"allow_profile_mismatch", "attempt to decode anyway if HW accelerated
> decoder's supported profiles do not exactly match the stream", 0,
> AV_OPT_TYPE_CONST, {.i64 =
> AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH }, INT_MIN, INT_MAX, V |
> D, "hwaccel_flags"}, {"extra_hw_frames", "Number of extra hardware frames
> to allocate for the user", OFFSET(extra_hw_frames), AV_OPT_TYPE_INT, { .i64
> = -1 }, -1, INT_MAX, V|D }, {"discard_damaged_percentage", "Percentage of
> damaged samples to discard a frame", OFFSET(discard_damaged_percentage),
> AV_OPT_TYPE_INT, {.i64 = 95 }, 0, 100, V|D },
> +{"sfc_flags", "set sfc flags", OFFSET(sfc_flags), AV_OPT_TYPE_INT, {
> +.i64 = 1 }, 0, 1, V|D, "sfc"}, {"sfc_format", "set sfc format",
> +OFFSET(sfc_format), AV_OPT_TYPE_PIXEL_FMT, {.i64=AV_PIX_FMT_ARGB}, -
> 1,
> +INT_MAX, V|D, "sfc"}, {"sfc_width", "set sfc width", OFFSET(sfc_width),
> +AV_OPT_TYPE_INT, {.i64 = 480}, 0, INT_MAX, V|D, "sfc"}, {"sfc_height",
> +"set sfc height", OFFSET(sfc_height), AV_OPT_TYPE_INT, {.i64 = 360}, 0,
> +INT_MAX, V|D, "sfc"},
> {NULL},
> };
>
> diff --git a/libavcodec/vaapi_decode.c b/libavcodec/vaapi_decode.c index
> 69512e1d45..2917ee951e 100644
> --- a/libavcodec/vaapi_decode.c
> +++ b/libavcodec/vaapi_decode.c
> @@ -145,6 +145,17 @@ static void
> ff_vaapi_decode_destroy_buffers(AVCodecContext *avctx,
> pic->slice_buffers[i], vas, vaErrorStr(vas));
> }
> }
> +
> + //destory sfc buffer
> + if (avctx->sfc_flags) {
> + vas = vaDestroyBuffer(ctx->hwctx->display,
> + pic->sfc_buffer);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to destroy sfc "
> + "buffer %#x: %d (%s).\n",
> + pic->sfc_buffer, vas, vaErrorStr(vas));
> + }
> + }
> }
>
> int ff_vaapi_decode_issue(AVCodecContext *avctx, @@ -184,6 +195,56 @@
> int ff_vaapi_decode_issue(AVCodecContext *avctx,
> goto fail_with_picture;
> }
>
> + //add sfc buffer
> + if (avctx->sfc_flags) {
> + VARectangle rect_src; /**< @brief Rectangle for source input */
> + VARectangle rect_sfc; /**< @brief Rectangle for SFC output */
> + VAProcPipelineParameterBuffer buffer;
> +
> + memset(&rect_src, 0, sizeof(rect_src));
> + memset(&rect_sfc, 0, sizeof(rect_sfc));
> + memset(&buffer, 0, sizeof(buffer));
> +
> + rect_src.x = rect_src.y = 0;
> + rect_src.width = (uint16_t)pic->sfc_src_width;
> + rect_src.height = (uint16_t)pic->sfc_src_height;
> +
> + rect_sfc.x = rect_src.y = 0;
> + rect_sfc.width = (uint16_t)avctx->sfc_width;
> + rect_sfc.height = (uint16_t)avctx->sfc_height;
> +
> + buffer.surface_region = &rect_src;
> + buffer.output_region = &rect_sfc;
> + buffer.additional_outputs = (VASurfaceID*)&(pic->sfc_output_surface);
> + buffer.num_additional_outputs = 1;
> +
> + vas = vaCreateBuffer(ctx->hwctx->display, ctx->va_context,
> + VAProcPipelineParameterBufferType,
> + sizeof(buffer),
> + 1,
> + (uint8_t*)&buffer,
> + &pic->sfc_buffer);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to create sfc buffer: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR(EIO);
> + goto fail_with_picture;
> + }
> +
> + av_log(avctx, AV_LOG_DEBUG, "H264 create sfc buffer.\n");
> +
> + vas = vaRenderPicture(ctx->hwctx->display, ctx->va_context,
> + &pic->sfc_buffer, 1);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to upload sfc: "
> + "%d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR(EIO);
> + goto fail_with_picture;
> + }
> +
> + av_log(avctx, AV_LOG_DEBUG, "H264 render sfc buffer.\n");
> + }
> +
> vas = vaEndPicture(ctx->hwctx->display, ctx->va_context);
> if (vas != VA_STATUS_SUCCESS) {
> av_log(avctx, AV_LOG_ERROR, "Failed to end picture decode "
> @@ -493,14 +554,49 @@ static int
> vaapi_decode_make_config(AVCodecContext *avctx,
> }
> }
>
> - vas = vaCreateConfig(hwctx->display, matched_va_profile,
> - VAEntrypointVLD, NULL, 0,
> - va_config);
> - if (vas != VA_STATUS_SUCCESS) {
> - av_log(avctx, AV_LOG_ERROR, "Failed to create decode "
> - "configuration: %d (%s).\n", vas, vaErrorStr(vas));
> - err = AVERROR(EIO);
> - goto fail;
> + if (avctx->sfc_flags) {
> + VAConfigAttrib attrib;
> +
> + memset(&attrib, 0, sizeof(attrib));
> + attrib.type = VAConfigAttribDecProcessing;
> + attrib.value = 0;
> +
> + vaGetConfigAttributes(
> + hwctx->display,
> + matched_va_profile,
> + VAEntrypointVLD,
> + &attrib,
> + 1);
> +
> + if (attrib.value != VA_DEC_PROCESSING) {
> + err = AVERROR(EINVAL);
> + goto fail;
> + }
> +
> + av_log(avctx, AV_LOG_DEBUG, "vaapi driver supports VDSFC\n");
> +
> + attrib.type = VAConfigAttribDecProcessing;
> + attrib.value = VA_DEC_PROCESSING;
> +
> + vas = vaCreateConfig(hwctx->display, matched_va_profile,
> + VAEntrypointVLD, &attrib, 1,
> + va_config);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to create decode "
> + "configuration: %d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR(EIO);
> + goto fail;
> + }
> + } else {
> + vas = vaCreateConfig(hwctx->display, matched_va_profile,
> + VAEntrypointVLD, NULL, 0,
> + va_config);
> + if (vas != VA_STATUS_SUCCESS) {
> + av_log(avctx, AV_LOG_ERROR, "Failed to create decode "
> + "configuration: %d (%s).\n", vas, vaErrorStr(vas));
> + err = AVERROR(EIO);
> + goto fail;
> + }
> }
>
> hwconfig = av_hwdevice_hwconfig_alloc(device_ref);
> @@ -587,6 +683,7 @@ int
> ff_vaapi_common_frame_params(AVCodecContext *avctx,
> AVBufferRef *hw_frames_ctx) {
> AVHWFramesContext *hw_frames = (AVHWFramesContext
> *)hw_frames_ctx->data;
> + AVVAAPIFramesContext *avfc = hw_frames->hwctx;
> AVHWDeviceContext *device_ctx = hw_frames->device_ctx;
> AVVAAPIDeviceContext *hwctx;
> VAConfigID va_config = VA_INVALID_ID; @@ -604,6 +701,15 @@ int
> ff_vaapi_common_frame_params(AVCodecContext *avctx,
> if (va_config != VA_INVALID_ID)
> vaDestroyConfig(hwctx->display, va_config);
>
> + if (avctx->sfc_flags) {
> + avfc->sfc_flags = avctx->sfc_flags;
> + avfc->sfc_format = avctx->sfc_format;
> + avfc->sfc_width = avctx->sfc_width;
> + avfc->sfc_height = avctx->sfc_height;
> +
> + av_log(avctx, AV_LOG_DEBUG, "VDSFC save sfc info avfc\n");
> + }
> +
> return 0;
> }
>
> diff --git a/libavcodec/vaapi_decode.h b/libavcodec/vaapi_decode.h index
> 6b415dd1d3..6674b496e1 100644
> --- a/libavcodec/vaapi_decode.h
> +++ b/libavcodec/vaapi_decode.h
> @@ -37,6 +37,29 @@ static inline VASurfaceID
> ff_vaapi_get_surface_id(AVFrame *pic)
> return (uintptr_t)pic->data[3];
> }
>
> +static inline VASurfaceID ff_vaapi_get_sfc_surface_id(AVFrame *pic) {
> + VASurfaceID sfc_surface_id = VA_INVALID_ID;
> +
> + AVFrameSideData *frame_sfc_sd = av_frame_get_side_data(pic,
> + AV_FRAME_DATA_SFC_INFO);
> + if (frame_sfc_sd) {
> + sfc_surface_id = (uintptr_t)frame_sfc_sd->buf->data;
> + }
> +
> + return sfc_surface_id;
> +}
> +
> +static inline int ff_vaapi_get_sfc_src_width(AVFrame *pic) {
> + return pic->width;
> +}
> +
> +static inline int ff_vaapi_get_sfc_src_height(AVFrame *pic) {
> + return pic->height;
> +}
> +
> enum {
> MAX_PARAM_BUFFERS = 16,
> };
> @@ -50,6 +73,13 @@ typedef struct VAAPIDecodePicture {
> int nb_slices;
> VABufferID *slice_buffers;
> int slices_allocated;
> +
> + //sfc info
> + VASurfaceID sfc_output_surface;
> + int sfc_src_width;
> + int sfc_src_height;
> + VABufferID sfc_buffer;
> +
> } VAAPIDecodePicture;
>
> typedef struct VAAPIDecodeContext {
> diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c index
> 5854587a25..a3f080b593 100644
> --- a/libavcodec/vaapi_h264.c
> +++ b/libavcodec/vaapi_h264.c
> @@ -302,6 +302,19 @@ static int vaapi_h264_start_frame(AVCodecContext
> *avctx,
> if (err < 0)
> goto fail;
>
> + //get sfc surface id
> + if (avctx->sfc_flags) {
> + pic->sfc_output_surface =
> + ff_vaapi_get_sfc_surface_id(h->cur_pic_ptr->f);
> +
> + //get sfc src width and height
> + pic->sfc_src_width = ff_vaapi_get_sfc_src_width(h->cur_pic_ptr->f);
> + pic->sfc_src_height =
> + ff_vaapi_get_sfc_src_height(h->cur_pic_ptr->f);
> +
> + av_log(avctx, AV_LOG_DEBUG, "H264 get sfc surface id: %d\n", pic-
> >sfc_output_surface);
> + av_log(avctx, AV_LOG_DEBUG, "H264 get sfc width: %d\n", pic-
> >sfc_src_width);
> + av_log(avctx, AV_LOG_DEBUG, "H264 get sfc height: %d\n", pic-
> >sfc_src_height);
> + }
> +
> return 0;
>
> fail:
Please ignore this patch, new version will be send out soon.
Sorry for inconvenience.
> --
> 2.17.1
More information about the ffmpeg-devel
mailing list