FFmpeg
vulkan_h264.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #include "h264dec.h"
20 #include "h264_ps.h"
21 
22 #include "vulkan_decode.h"
23 
26  .decode_extension = FF_VK_EXT_VIDEO_DECODE_H264,
27  .decode_op = VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR,
28  .ext_props = {
29  .extensionName = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_EXTENSION_NAME,
30  .specVersion = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_SPEC_VERSION,
31  },
32 };
33 
34 typedef struct H264VulkanDecodePicture {
36 
37  /* Current picture */
38  StdVideoDecodeH264ReferenceInfo h264_ref;
39  VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref;
40 
41  /* Picture refs */
43  StdVideoDecodeH264ReferenceInfo h264_refs [H264_MAX_PICTURE_COUNT];
44  VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT];
45 
46  /* Current picture (contd.) */
47  StdVideoDecodeH264PictureInfo h264pic;
48  VkVideoDecodeH264PictureInfoKHR h264_pic_info;
50 
51 const static int h264_scaling_list8_order[] = { 0, 3, 1, 4, 2, 5 };
52 
53 static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src,
54  VkVideoReferenceSlotInfoKHR *ref_slot, /* Main structure */
55  VkVideoPictureResourceInfoKHR *ref, /* Goes in ^ */
56  VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, /* Goes in ^ */
57  StdVideoDecodeH264ReferenceInfo *h264_ref, /* Goes in ^ */
58  H264Picture *pic, int is_current,
59  int is_field, int picture_structure,
60  int dpb_slot_index)
61 {
64  FFVulkanDecodePicture *vkpic = &hp->vp;
65 
66  int err = ff_vk_decode_prepare_frame(dec, pic->f, vkpic, is_current,
67  dec->dedicated_dpb);
68  if (err < 0)
69  return err;
70 
71  *h264_ref = (StdVideoDecodeH264ReferenceInfo) {
72  .FrameNum = pic->long_ref ? pic->pic_id : pic->frame_num,
73  .PicOrderCnt = { pic->field_poc[0], pic->field_poc[1] },
74  .flags = (StdVideoDecodeH264ReferenceInfoFlags) {
75  .top_field_flag = is_field ? !!(picture_structure & PICT_TOP_FIELD) : 0,
76  .bottom_field_flag = is_field ? !!(picture_structure & PICT_BOTTOM_FIELD) : 0,
77  .used_for_long_term_reference = pic->reference && pic->long_ref,
78  /*
79  * flags.is_non_existing is used to indicate whether the picture is marked as
80  * “non-existing” as defined in section 8.2.5.2 of the ITU-T H.264 Specification;
81  * 8.2.5.2 Decoding process for gaps in frame_num
82  * corresponds to the code in h264_slice.c:h264_field_start,
83  * which sets the invalid_gap flag when decoding.
84  */
85  .is_non_existing = pic->invalid_gap,
86  },
87  };
88 
89  *vkh264_ref = (VkVideoDecodeH264DpbSlotInfoKHR) {
90  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR,
91  .pStdReferenceInfo = h264_ref,
92  };
93 
94  *ref = (VkVideoPictureResourceInfoKHR) {
95  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
96  .codedOffset = (VkOffset2D){ 0, 0 },
97  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
98  .baseArrayLayer = dec->layered_dpb ? dpb_slot_index : 0,
99  .imageViewBinding = vkpic->img_view_ref,
100  };
101 
102  *ref_slot = (VkVideoReferenceSlotInfoKHR) {
103  .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
104  .pNext = vkh264_ref,
105  .slotIndex = dpb_slot_index,
106  .pPictureResource = ref,
107  };
108 
109  if (ref_src)
110  *ref_src = pic;
111 
112  return 0;
113 }
114 
115 static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
116 {
117  switch (level_idc) {
118  case 10: return STD_VIDEO_H264_LEVEL_IDC_1_0;
119  case 11: return STD_VIDEO_H264_LEVEL_IDC_1_1;
120  case 12: return STD_VIDEO_H264_LEVEL_IDC_1_2;
121  case 13: return STD_VIDEO_H264_LEVEL_IDC_1_3;
122  case 20: return STD_VIDEO_H264_LEVEL_IDC_2_0;
123  case 21: return STD_VIDEO_H264_LEVEL_IDC_2_1;
124  case 22: return STD_VIDEO_H264_LEVEL_IDC_2_2;
125  case 30: return STD_VIDEO_H264_LEVEL_IDC_3_0;
126  case 31: return STD_VIDEO_H264_LEVEL_IDC_3_1;
127  case 32: return STD_VIDEO_H264_LEVEL_IDC_3_2;
128  case 40: return STD_VIDEO_H264_LEVEL_IDC_4_0;
129  case 41: return STD_VIDEO_H264_LEVEL_IDC_4_1;
130  case 42: return STD_VIDEO_H264_LEVEL_IDC_4_2;
131  case 50: return STD_VIDEO_H264_LEVEL_IDC_5_0;
132  case 51: return STD_VIDEO_H264_LEVEL_IDC_5_1;
133  case 52: return STD_VIDEO_H264_LEVEL_IDC_5_2;
134  case 60: return STD_VIDEO_H264_LEVEL_IDC_6_0;
135  case 61: return STD_VIDEO_H264_LEVEL_IDC_6_1;
136  default:
137  case 62: return STD_VIDEO_H264_LEVEL_IDC_6_2;
138  }
139 }
140 
141 static void set_sps(const SPS *sps,
142  StdVideoH264ScalingLists *vksps_scaling,
143  StdVideoH264HrdParameters *vksps_vui_header,
144  StdVideoH264SequenceParameterSetVui *vksps_vui,
145  StdVideoH264SequenceParameterSet *vksps)
146 {
147  *vksps_scaling = (StdVideoH264ScalingLists) {
148  .scaling_list_present_mask = sps->scaling_matrix_present_mask,
149  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
150  };
151 
152  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
153  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
154  vksps_scaling->ScalingList4x4[i][j] = sps->scaling_matrix4[i][ff_zigzag_scan[j]];
155 
156  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
157  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
158  vksps_scaling->ScalingList8x8[i][j] =
159  sps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
160 
161  *vksps_vui_header = (StdVideoH264HrdParameters) {
162  .cpb_cnt_minus1 = sps->cpb_cnt - 1,
163  .bit_rate_scale = sps->bit_rate_scale,
164  .initial_cpb_removal_delay_length_minus1 = sps->initial_cpb_removal_delay_length - 1,
165  .cpb_removal_delay_length_minus1 = sps->cpb_removal_delay_length - 1,
166  .dpb_output_delay_length_minus1 = sps->dpb_output_delay_length - 1,
167  .time_offset_length = sps->time_offset_length,
168  };
169 
170  for (int i = 0; i < sps->cpb_cnt; i++) {
171  vksps_vui_header->bit_rate_value_minus1[i] = sps->bit_rate_value[i] - 1;
172  vksps_vui_header->cpb_size_value_minus1[i] = sps->cpb_size_value[i] - 1;
173  vksps_vui_header->cbr_flag[i] = (sps->cpr_flag >> i) & 0x1;
174  }
175 
176  *vksps_vui = (StdVideoH264SequenceParameterSetVui) {
177  .aspect_ratio_idc = sps->vui.aspect_ratio_idc,
178  .sar_width = sps->vui.sar.num,
179  .sar_height = sps->vui.sar.den,
180  .video_format = sps->vui.video_format,
181  .colour_primaries = sps->vui.colour_primaries,
182  .transfer_characteristics = sps->vui.transfer_characteristics,
183  .matrix_coefficients = sps->vui.matrix_coeffs,
184  .num_units_in_tick = sps->num_units_in_tick,
185  .time_scale = sps->time_scale,
186  .pHrdParameters = vksps_vui_header,
187  .max_num_reorder_frames = sps->num_reorder_frames,
188  .max_dec_frame_buffering = sps->max_dec_frame_buffering,
189  .flags = (StdVideoH264SpsVuiFlags) {
190  .aspect_ratio_info_present_flag = sps->vui.aspect_ratio_info_present_flag,
191  .overscan_info_present_flag = sps->vui.overscan_info_present_flag,
192  .overscan_appropriate_flag = sps->vui.overscan_appropriate_flag,
193  .video_signal_type_present_flag = sps->vui.video_signal_type_present_flag,
194  .video_full_range_flag = sps->vui.video_full_range_flag,
195  .color_description_present_flag = sps->vui.colour_description_present_flag,
196  .chroma_loc_info_present_flag = sps->vui.chroma_location,
197  .timing_info_present_flag = sps->timing_info_present_flag,
198  .fixed_frame_rate_flag = sps->fixed_frame_rate_flag,
199  .bitstream_restriction_flag = sps->bitstream_restriction_flag,
200  .nal_hrd_parameters_present_flag = sps->nal_hrd_parameters_present_flag,
201  .vcl_hrd_parameters_present_flag = sps->vcl_hrd_parameters_present_flag,
202  },
203  };
204 
205  *vksps = (StdVideoH264SequenceParameterSet) {
206  .profile_idc = sps->profile_idc,
207  .level_idc = convert_to_vk_level_idc(sps->level_idc),
208  .seq_parameter_set_id = sps->sps_id,
209  .chroma_format_idc = sps->chroma_format_idc,
210  .bit_depth_luma_minus8 = sps->bit_depth_luma - 8,
211  .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8,
212  .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4,
213  .pic_order_cnt_type = sps->poc_type,
214  .log2_max_pic_order_cnt_lsb_minus4 = sps->poc_type ? 0 : sps->log2_max_poc_lsb - 4,
215  .offset_for_non_ref_pic = sps->offset_for_non_ref_pic,
216  .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field,
217  .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length,
218  .max_num_ref_frames = sps->ref_frame_count,
219  .pic_width_in_mbs_minus1 = sps->mb_width - 1,
220  .pic_height_in_map_units_minus1 = (sps->mb_height/(2 - sps->frame_mbs_only_flag)) - 1,
221  .frame_crop_left_offset = sps->crop_left,
222  .frame_crop_right_offset = sps->crop_right,
223  .frame_crop_top_offset = sps->crop_top,
224  .frame_crop_bottom_offset = sps->crop_bottom,
225  .flags = (StdVideoH264SpsFlags) {
226  .constraint_set0_flag = (sps->constraint_set_flags >> 0) & 0x1,
227  .constraint_set1_flag = (sps->constraint_set_flags >> 1) & 0x1,
228  .constraint_set2_flag = (sps->constraint_set_flags >> 2) & 0x1,
229  .constraint_set3_flag = (sps->constraint_set_flags >> 3) & 0x1,
230  .constraint_set4_flag = (sps->constraint_set_flags >> 4) & 0x1,
231  .constraint_set5_flag = (sps->constraint_set_flags >> 5) & 0x1,
232  .direct_8x8_inference_flag = sps->direct_8x8_inference_flag,
233  .mb_adaptive_frame_field_flag = sps->mb_aff,
234  .frame_mbs_only_flag = sps->frame_mbs_only_flag,
235  .delta_pic_order_always_zero_flag = sps->delta_pic_order_always_zero_flag,
236  .separate_colour_plane_flag = sps->residual_color_transform_flag,
237  .gaps_in_frame_num_value_allowed_flag = sps->gaps_in_frame_num_allowed_flag,
238  .qpprime_y_zero_transform_bypass_flag = sps->transform_bypass,
239  .frame_cropping_flag = sps->crop,
240  .seq_scaling_matrix_present_flag = sps->scaling_matrix_present,
241  .vui_parameters_present_flag = sps->vui_parameters_present_flag,
242  },
243  .pOffsetForRefFrame = sps->offset_for_ref_frame,
244  .pScalingLists = vksps_scaling,
245  .pSequenceParameterSetVui = vksps_vui,
246  };
247 }
248 
249 static void set_pps(const PPS *pps, const SPS *sps,
250  StdVideoH264ScalingLists *vkpps_scaling,
251  StdVideoH264PictureParameterSet *vkpps)
252 {
253  *vkpps_scaling = (StdVideoH264ScalingLists) {
254  .scaling_list_present_mask = pps->pic_scaling_matrix_present_mask,
255  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
256  };
257 
258  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
259  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
260  vkpps_scaling->ScalingList4x4[i][j] = pps->scaling_matrix4[i][ff_zigzag_scan[j]];
261 
262  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
263  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
264  vkpps_scaling->ScalingList8x8[i][j] =
265  pps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
266 
267  *vkpps = (StdVideoH264PictureParameterSet) {
268  .seq_parameter_set_id = pps->sps_id,
269  .pic_parameter_set_id = pps->pps_id,
270  .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1,
271  .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1,
272  .weighted_bipred_idc = pps->weighted_bipred_idc,
273  .pic_init_qp_minus26 = pps->init_qp - 26,
274  .pic_init_qs_minus26 = pps->init_qs - 26,
275  .chroma_qp_index_offset = pps->chroma_qp_index_offset[0],
276  .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1],
277  .flags = (StdVideoH264PpsFlags) {
278  .transform_8x8_mode_flag = pps->transform_8x8_mode,
279  .redundant_pic_cnt_present_flag = pps->redundant_pic_cnt_present,
280  .constrained_intra_pred_flag = pps->constrained_intra_pred,
281  .deblocking_filter_control_present_flag = pps->deblocking_filter_parameters_present,
282  .weighted_pred_flag = pps->weighted_pred,
283  .bottom_field_pic_order_in_frame_present_flag = pps->pic_order_present,
284  .entropy_coding_mode_flag = pps->cabac,
285  .pic_scaling_matrix_present_flag = pps->pic_scaling_matrix_present_flag,
286  },
287  .pScalingLists = vkpps_scaling,
288  };
289 }
290 
292 {
293  int err;
296  const H264Context *h = avctx->priv_data;
297 
298  /* SPS */
299  StdVideoH264ScalingLists vksps_scaling[MAX_SPS_COUNT];
300  StdVideoH264HrdParameters vksps_vui_header[MAX_SPS_COUNT];
301  StdVideoH264SequenceParameterSetVui vksps_vui[MAX_SPS_COUNT];
302  StdVideoH264SequenceParameterSet vksps[MAX_SPS_COUNT];
303 
304  /* PPS */
305  StdVideoH264ScalingLists vkpps_scaling[MAX_PPS_COUNT];
306  StdVideoH264PictureParameterSet vkpps[MAX_PPS_COUNT];
307 
308  VkVideoDecodeH264SessionParametersAddInfoKHR h264_params_info = {
309  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_ADD_INFO_KHR,
310  .pStdSPSs = vksps,
311  .stdSPSCount = 0,
312  .pStdPPSs = vkpps,
313  .stdPPSCount = 0,
314  };
315  VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
316  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
317  .pParametersAddInfo = &h264_params_info,
318  };
319  VkVideoSessionParametersCreateInfoKHR session_params_create = {
320  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
321  .pNext = &h264_params,
322  .videoSession = ctx->common.session,
323  .videoSessionParametersTemplate = VK_NULL_HANDLE,
324  };
325 
326  /* SPS list */
327  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.sps_list); i++) {
328  if (h->ps.sps_list[i]) {
329  const SPS *sps_l = h->ps.sps_list[i];
330  int idx = h264_params_info.stdSPSCount;
331  set_sps(sps_l, &vksps_scaling[idx], &vksps_vui_header[idx], &vksps_vui[idx], &vksps[idx]);
332  h264_params_info.stdSPSCount++;
333  }
334  }
335 
336  /* PPS list */
337  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.pps_list); i++) {
338  if (h->ps.pps_list[i]) {
339  const PPS *pps_l = h->ps.pps_list[i];
340  int idx = h264_params_info.stdPPSCount;
341  set_pps(pps_l, pps_l->sps, &vkpps_scaling[idx], &vkpps[idx]);
342  h264_params_info.stdPPSCount++;
343  }
344  }
345 
346  h264_params.maxStdSPSCount = h264_params_info.stdSPSCount;
347  h264_params.maxStdPPSCount = h264_params_info.stdPPSCount;
348 
349  err = ff_vk_decode_create_params(buf, avctx, ctx, &session_params_create);
350  if (err < 0)
351  return err;
352 
353  av_log(avctx, AV_LOG_DEBUG, "Created frame parameters: %i SPS %i PPS\n",
354  h264_params_info.stdSPSCount, h264_params_info.stdPPSCount);
355 
356  return 0;
357 }
358 
360  av_unused const uint8_t *buffer,
361  av_unused uint32_t size)
362 {
363  int err;
364  int dpb_slot_index = 0;
365  H264Context *h = avctx->priv_data;
366  H264Picture *pic = h->cur_pic_ptr;
369  FFVulkanDecodePicture *vp = &hp->vp;
370 
371  if (!dec->session_params) {
372  err = vk_h264_create_params(avctx, &dec->session_params);
373  if (err < 0)
374  return err;
375  }
376 
377  /* Fill in main slot */
378  dpb_slot_index = 0;
379  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
380  if (pic == &h->DPB[slot]) {
381  dpb_slot_index = slot;
382  break;
383  }
384  }
385 
386  err = vk_h264_fill_pict(avctx, NULL, &vp->ref_slot, &vp->ref,
387  &hp->vkh264_ref, &hp->h264_ref, pic, 1,
388  h->DPB[dpb_slot_index].field_picture,
389  h->DPB[dpb_slot_index].reference,
390  dpb_slot_index);
391  if (err < 0)
392  return err;
393 
394  /* Fill in short-term references */
395  for (int i = 0; i < h->short_ref_count; i++) {
396  dpb_slot_index = 0;
397  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
398  if (h->short_ref[i] == &h->DPB[slot]) {
399  dpb_slot_index = slot;
400  break;
401  }
402  }
403  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
404  &vp->refs[i], &hp->vkh264_refs[i],
405  &hp->h264_refs[i], h->short_ref[i], 0,
406  h->DPB[dpb_slot_index].field_picture,
407  h->DPB[dpb_slot_index].reference,
408  dpb_slot_index);
409  if (err < 0)
410  return err;
411  }
412 
413  /* Fill in long-term refs */
414  for (int r = 0, i = h->short_ref_count; r < H264_MAX_DPB_FRAMES &&
415  i < h->short_ref_count + h->long_ref_count; r++) {
416  if (!h->long_ref[r])
417  continue;
418 
419  dpb_slot_index = 0;
420  for (unsigned slot = 0; slot < 16; slot++) {
421  if (h->long_ref[r] == &h->DPB[slot]) {
422  dpb_slot_index = slot;
423  break;
424  }
425  }
426  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
427  &vp->refs[i], &hp->vkh264_refs[i],
428  &hp->h264_refs[i], h->long_ref[r], 0,
429  h->DPB[dpb_slot_index].field_picture,
430  h->DPB[dpb_slot_index].reference,
431  dpb_slot_index);
432  if (err < 0)
433  return err;
434  i++;
435  }
436 
437  hp->h264pic = (StdVideoDecodeH264PictureInfo) {
438  .seq_parameter_set_id = pic->pps->sps_id,
439  .pic_parameter_set_id = pic->pps->pps_id,
440  .frame_num = 0, /* Set later */
441  .idr_pic_id = 0, /* Set later */
442  .PicOrderCnt[0] = pic->field_poc[0],
443  .PicOrderCnt[1] = pic->field_poc[1],
444  .flags = (StdVideoDecodeH264PictureInfoFlags) {
445  .field_pic_flag = FIELD_PICTURE(h),
446  .is_intra = 1, /* Set later */
447  .IdrPicFlag = h->picture_idr,
448  .bottom_field_flag = h->picture_structure != PICT_FRAME &&
449  h->picture_structure & PICT_BOTTOM_FIELD,
450  .is_reference = h->nal_ref_idc != 0,
451  .complementary_field_pair = h->first_field && FIELD_PICTURE(h),
452  },
453  };
454 
455  hp->h264_pic_info = (VkVideoDecodeH264PictureInfoKHR) {
456  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PICTURE_INFO_KHR,
457  .pStdPictureInfo = &hp->h264pic,
458  };
459 
460  vp->decode_info = (VkVideoDecodeInfoKHR) {
461  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_INFO_KHR,
462  .pNext = &hp->h264_pic_info,
463  .flags = 0x0,
464  .pSetupReferenceSlot = &vp->ref_slot,
465  .referenceSlotCount = h->short_ref_count + h->long_ref_count,
466  .pReferenceSlots = vp->ref_slots,
467  .dstPictureResource = (VkVideoPictureResourceInfoKHR) {
468  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
469  .codedOffset = (VkOffset2D){ 0, 0 },
470  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
471  .baseArrayLayer = 0,
472  .imageViewBinding = vp->img_view_out,
473  },
474  };
475 
476  return 0;
477 }
478 
480  const uint8_t *data,
481  uint32_t size)
482 {
483  const H264Context *h = avctx->priv_data;
484  const H264SliceContext *sl = &h->slice_ctx[0];
485  H264VulkanDecodePicture *hp = h->cur_pic_ptr->hwaccel_picture_private;
486  FFVulkanDecodePicture *vp = &hp->vp;
487 
488  int err = ff_vk_decode_add_slice(avctx, vp, data, size, 1,
489  &hp->h264_pic_info.sliceCount,
490  &hp->h264_pic_info.pSliceOffsets);
491  if (err < 0)
492  return err;
493 
494  hp->h264pic.frame_num = sl->frame_num;
495  hp->h264pic.idr_pic_id = sl->idr_pic_id;
496 
497  /* Frame is only intra of all slices are marked as intra */
499  hp->h264pic.flags.is_intra = 0;
500 
501  return 0;
502 }
503 
505 {
506  const H264Context *h = avctx->priv_data;
507  H264Picture *pic = h->cur_pic_ptr;
510  FFVulkanDecodePicture *vp = &hp->vp;
512  AVFrame *rav[H264_MAX_PICTURE_COUNT] = { 0 };
513 
514  if (!hp->h264_pic_info.sliceCount)
515  return 0;
516 
517  if (!vp->slices_buf)
518  return AVERROR(EINVAL);
519 
520  if (!dec->session_params) {
521  int err = vk_h264_create_params(avctx, &dec->session_params);
522  if (err < 0)
523  return err;
524 
525  hp->h264pic.seq_parameter_set_id = pic->pps->sps_id;
526  hp->h264pic.pic_parameter_set_id = pic->pps->pps_id;
527  }
528 
529  for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
530  H264Picture *rp = hp->ref_src[i];
532 
533  rvp[i] = &rhp->vp;
534  rav[i] = hp->ref_src[i]->f;
535  }
536 
537  av_log(avctx, AV_LOG_VERBOSE, "Decoding frame, %"SIZE_SPECIFIER" bytes, %i slices\n",
538  vp->slices_size, hp->h264_pic_info.sliceCount);
539 
540  return ff_vk_decode_frame(avctx, pic->f, vp, rav, rvp);
541 }
542 
544 {
545  AVHWDeviceContext *hwctx = _hwctx.nc;
547 
548  /* Free frame resources, this also destroys the session parameters. */
549  ff_vk_decode_free_frame(hwctx, &hp->vp);
550 }
551 
553  .p.name = "h264_vulkan",
554  .p.type = AVMEDIA_TYPE_VIDEO,
555  .p.id = AV_CODEC_ID_H264,
556  .p.pix_fmt = AV_PIX_FMT_VULKAN,
557  .start_frame = &vk_h264_start_frame,
558  .decode_slice = &vk_h264_decode_slice,
559  .end_frame = &vk_h264_end_frame,
560  .free_frame_priv = &vk_h264_free_frame_priv,
561  .frame_priv_data_size = sizeof(H264VulkanDecodePicture),
564  .decode_params = &ff_vk_params_invalidate,
567  .frame_params = &ff_vk_frame_params,
568  .priv_data_size = sizeof(FFVulkanDecodeContext),
570 };
PICT_FRAME
#define PICT_FRAME
Definition: mpegutils.h:33
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:112
r
const char * r
Definition: vf_curves.c:127
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
FF_VK_EXT_VIDEO_DECODE_H264
@ FF_VK_EXT_VIDEO_DECODE_H264
Definition: vulkan_functions.h:44
H264Picture::f
AVFrame * f
Definition: h264dec.h:107
H264VulkanDecodePicture::h264_ref
StdVideoDecodeH264ReferenceInfo h264_ref
Definition: vulkan_h264.c:38
set_sps
static void set_sps(const SPS *sps, StdVideoH264ScalingLists *vksps_scaling, StdVideoH264HrdParameters *vksps_vui_header, StdVideoH264SequenceParameterSetVui *vksps_vui, StdVideoH264SequenceParameterSet *vksps)
Definition: vulkan_h264.c:141
FFVulkanDecodeContext::shared_ctx
FFVulkanDecodeShared * shared_ctx
Definition: vulkan_decode.h:66
H264VulkanDecodePicture::h264_pic_info
VkVideoDecodeH264PictureInfoKHR h264_pic_info
Definition: vulkan_h264.c:48
av_unused
#define av_unused
Definition: attributes.h:131
H264Picture::pps
const PPS * pps
Definition: h264dec.h:150
FFHWAccel::p
AVHWAccel p
The public AVHWAccel.
Definition: hwaccel_internal.h:38
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
AVFrame::width
int width
Definition: frame.h:446
level_idc
int level_idc
Definition: h264_levels.c:29
set_pps
static void set_pps(const PPS *pps, const SPS *sps, StdVideoH264ScalingLists *vkpps_scaling, StdVideoH264PictureParameterSet *vkpps)
Definition: vulkan_h264.c:249
data
const char data[16]
Definition: mxf.c:148
FFVulkanDecodeDescriptor::codec_id
enum AVCodecID codec_id
Definition: vulkan_decode.h:30
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
PICT_BOTTOM_FIELD
#define PICT_BOTTOM_FIELD
Definition: mpegutils.h:32
H264VulkanDecodePicture::vkh264_ref
VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref
Definition: vulkan_h264.c:39
H264VulkanDecodePicture::h264pic
StdVideoDecodeH264PictureInfo h264pic
Definition: vulkan_h264.c:47
FFRefStructOpaque
RefStruct is an API for creating reference-counted objects with minimal overhead.
Definition: refstruct.h:58
vk_h264_create_params
static int vk_h264_create_params(AVCodecContext *avctx, AVBufferRef **buf)
Definition: vulkan_h264.c:291
H264Picture::invalid_gap
int invalid_gap
Definition: h264dec.h:146
FFVulkanDecodeContext
Definition: vulkan_decode.h:65
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:187
FFVulkanDecodePicture::ref
VkVideoPictureResourceInfoKHR ref
Definition: vulkan_decode.h:100
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:67
H264Picture::frame_num
int frame_num
frame_num (raw frame_num from slice header)
Definition: h264dec.h:128
H264SliceContext
Definition: h264dec.h:172
ff_h264_vulkan_hwaccel
const FFHWAccel ff_h264_vulkan_hwaccel
Definition: vulkan_h264.c:552
FFHWAccel
Definition: hwaccel_internal.h:34
HWACCEL_CAP_THREAD_SAFE
#define HWACCEL_CAP_THREAD_SAFE
Definition: hwaccel_internal.h:32
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:343
FFVulkanDecodeShared
Definition: vulkan_decode.h:46
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:60
vk_h264_free_frame_priv
static void vk_h264_free_frame_priv(FFRefStructOpaque _hwctx, void *data)
Definition: vulkan_h264.c:543
FFVulkanDecodePicture::img_view_out
VkImageView img_view_out
Definition: vulkan_decode.h:91
H264_MAX_DPB_FRAMES
@ H264_MAX_DPB_FRAMES
Definition: h264.h:76
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
vk_h264_decode_slice
static int vk_h264_decode_slice(AVCodecContext *avctx, const uint8_t *data, uint32_t size)
Definition: vulkan_h264.c:479
FFVulkanDecodePicture::refs
VkVideoPictureResourceInfoKHR refs[36]
Definition: vulkan_decode.h:104
H264VulkanDecodePicture
Definition: vulkan_h264.c:34
FFVulkanDecodePicture
Definition: vulkan_decode.h:87
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1064
PPS::pps_id
unsigned int pps_id
Definition: h264_ps.h:111
H264VulkanDecodePicture::h264_refs
StdVideoDecodeH264ReferenceInfo h264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:43
FIELD_PICTURE
#define FIELD_PICTURE(h)
Definition: h264dec.h:67
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:49
PICT_TOP_FIELD
#define PICT_TOP_FIELD
Definition: mpegutils.h:31
H264VulkanDecodePicture::ref_src
H264Picture * ref_src[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:42
H264SliceContext::slice_type
int slice_type
Definition: h264dec.h:178
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
HWACCEL_CAP_ASYNC_SAFE
#define HWACCEL_CAP_ASYNC_SAFE
Header providing the internals of AVHWAccel.
Definition: hwaccel_internal.h:31
FFVulkanDecodePicture::img_view_ref
VkImageView img_view_ref
Definition: vulkan_decode.h:90
NULL
#define NULL
Definition: coverity.c:32
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:573
convert_to_vk_level_idc
static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
Definition: vulkan_h264.c:115
SPS
Sequence parameter set.
Definition: h264_ps.h:44
AVCodecContext::internal
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:480
AV_PICTURE_TYPE_SI
@ AV_PICTURE_TYPE_SI
Switching Intra.
Definition: avutil.h:283
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:279
PPS
Picture parameter set.
Definition: h264_ps.h:110
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1093
flush
void(* flush)(AVBSFContext *ctx)
Definition: dts2pts.c:368
MAX_PPS_COUNT
#define MAX_PPS_COUNT
Definition: h264_ps.h:38
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:1007
h264_ps.h
H264Picture::pic_id
int pic_id
pic_num (short -> no wrap version of pic_num, pic_num & max_pic_num; long -> long_pic_num)
Definition: h264dec.h:131
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:366
ff_zigzag_scan
const uint8_t ff_zigzag_scan[16+1]
Definition: mathtables.c:109
H264Picture::reference
int reference
Definition: h264dec.h:144
AVCodecInternal::hwaccel_priv_data
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:123
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
size
int size
Definition: twinvq_data.h:10344
h264_scaling_list8_order
const static int h264_scaling_list8_order[]
Definition: vulkan_h264.c:51
FFRefStructOpaque::nc
void * nc
Definition: refstruct.h:59
ff_vk_dec_h264_desc
const FFVulkanDecodeDescriptor ff_vk_dec_h264_desc
Definition: vulkan_h264.c:24
AVHWAccel::name
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:2095
h264dec.h
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
H264Context
H264Context.
Definition: h264dec.h:332
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
vk_h264_end_frame
static int vk_h264_end_frame(AVCodecContext *avctx)
Definition: vulkan_h264.c:504
H264SliceContext::frame_num
int frame_num
Definition: h264dec.h:320
H264VulkanDecodePicture::vp
FFVulkanDecodePicture vp
Definition: vulkan_h264.c:35
vk_h264_start_frame
static int vk_h264_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
Definition: vulkan_h264.c:359
FFVulkanDecodePicture::ref_slot
VkVideoReferenceSlotInfoKHR ref_slot
Definition: vulkan_decode.h:101
FFVulkanDecodePicture::ref_slots
VkVideoReferenceSlotInfoKHR ref_slots[36]
Definition: vulkan_decode.h:105
ff_zigzag_direct
const uint8_t ff_zigzag_direct[64]
Definition: mathtables.c:98
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:313
PPS::sps
const SPS * sps
RefStruct reference.
Definition: h264_ps.h:141
vk_h264_fill_pict
static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src, VkVideoReferenceSlotInfoKHR *ref_slot, VkVideoPictureResourceInfoKHR *ref, VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, StdVideoDecodeH264ReferenceInfo *h264_ref, H264Picture *pic, int is_current, int is_field, int picture_structure, int dpb_slot_index)
Definition: vulkan_h264.c:53
sps
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
Definition: cbs_h264_syntax_template.c:260
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:251
SIZE_SPECIFIER
#define SIZE_SPECIFIER
Definition: internal.h:129
PPS::sps_id
unsigned int sps_id
Definition: h264_ps.h:112
AVCodecContext
main external API structure.
Definition: avcodec.h:445
AVFrame::height
int height
Definition: frame.h:446
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:70
H264Picture::field_poc
int field_poc[2]
top/bottom POC
Definition: h264dec.h:126
buffer
the frame and frame reference mechanism is intended to as much as expensive copies of that data while still allowing the filters to produce correct results The data is stored in buffers represented by AVFrame structures Several references can point to the same frame buffer
Definition: filter_design.txt:49
H264Picture
Definition: h264dec.h:106
update_thread_context
the pkt_dts and pkt_pts fields in AVFrame will work as usual Restrictions on codec whose streams don t reset across will not work because their bitstreams cannot be decoded in parallel *The contents of buffers must not be read before as well as code calling up to before the decode process starts Call have update_thread_context() run it in the next thread. Add AV_CODEC_CAP_FRAME_THREADS to the codec capabilities. There will be very little speed gain at this point but it should work. Use ff_thread_get_buffer()(or ff_progress_frame_get_buffer() in case you have inter-frame dependencies and use the ProgressFrame API) to allocate frame buffers. Call ff_progress_frame_report() after some part of the current picture has decoded. A good place to put this is where draw_horiz_band() is called - add this if it isn 't called anywhere
FFVulkanDecodeDescriptor
Definition: vulkan_decode.h:29
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
pps
uint64_t pps
Definition: dovi_rpuenc.c:35
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:121
MAX_SPS_COUNT
#define MAX_SPS_COUNT
Definition: h264_ps.h:37
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:80
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:111
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
H264Picture::hwaccel_picture_private
void * hwaccel_picture_private
RefStruct reference for hardware accelerator private data.
Definition: h264dec.h:122
vulkan_decode.h
H264SliceContext::idr_pic_id
int idr_pic_id
Definition: h264dec.h:321
AVCodecContext::priv_data
void * priv_data
Definition: avcodec.h:472
FFVulkanDecodeContext::layered_dpb
int layered_dpb
Definition: vulkan_decode.h:71
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1108
h
h
Definition: vp9dsp_template.c:2038
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:108
H264_MAX_PICTURE_COUNT
#define H264_MAX_PICTURE_COUNT
Definition: h264dec.h:49
H264VulkanDecodePicture::vkh264_refs
VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:44
H264Picture::long_ref
int long_ref
1->long term reference 0->short term reference
Definition: h264dec.h:133