FFmpeg
vulkan_h264.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #include "h264dec.h"
20 #include "h264_ps.h"
21 
22 #include "vulkan_decode.h"
23 
24 const VkExtensionProperties ff_vk_dec_h264_ext = {
25  .extensionName = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_EXTENSION_NAME,
26  .specVersion = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_SPEC_VERSION,
27 };
28 
29 typedef struct H264VulkanDecodePicture {
31 
32  /* Current picture */
33  StdVideoDecodeH264ReferenceInfo h264_ref;
34  VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref;
35 
36  /* Picture refs */
38  StdVideoDecodeH264ReferenceInfo h264_refs [H264_MAX_PICTURE_COUNT];
39  VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT];
40 
41  /* Current picture (contd.) */
42  StdVideoDecodeH264PictureInfo h264pic;
43  VkVideoDecodeH264PictureInfoKHR h264_pic_info;
45 
46 const static int h264_scaling_list8_order[] = { 0, 3, 1, 4, 2, 5 };
47 
48 static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src,
49  VkVideoReferenceSlotInfoKHR *ref_slot, /* Main structure */
50  VkVideoPictureResourceInfoKHR *ref, /* Goes in ^ */
51  VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, /* Goes in ^ */
52  StdVideoDecodeH264ReferenceInfo *h264_ref, /* Goes in ^ */
53  H264Picture *pic, int is_current,
54  int is_field, int picture_structure,
55  int dpb_slot_index)
56 {
59  FFVulkanDecodePicture *vkpic = &hp->vp;
60 
61  int err = ff_vk_decode_prepare_frame(dec, pic->f, vkpic, is_current,
62  dec->dedicated_dpb);
63  if (err < 0)
64  return err;
65 
66  *h264_ref = (StdVideoDecodeH264ReferenceInfo) {
67  .FrameNum = pic->long_ref ? pic->pic_id : pic->frame_num,
68  .PicOrderCnt = { pic->field_poc[0], pic->field_poc[1] },
69  .flags = (StdVideoDecodeH264ReferenceInfoFlags) {
70  .top_field_flag = is_field ? !!(picture_structure & PICT_TOP_FIELD) : 0,
71  .bottom_field_flag = is_field ? !!(picture_structure & PICT_BOTTOM_FIELD) : 0,
72  .used_for_long_term_reference = pic->reference && pic->long_ref,
73  /*
74  * flags.is_non_existing is used to indicate whether the picture is marked as
75  * “non-existing” as defined in section 8.2.5.2 of the ITU-T H.264 Specification;
76  * 8.2.5.2 Decoding process for gaps in frame_num
77  * corresponds to the code in h264_slice.c:h264_field_start,
78  * which sets the invalid_gap flag when decoding.
79  */
80  .is_non_existing = pic->invalid_gap,
81  },
82  };
83 
84  *vkh264_ref = (VkVideoDecodeH264DpbSlotInfoKHR) {
85  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR,
86  .pStdReferenceInfo = h264_ref,
87  };
88 
89  *ref = (VkVideoPictureResourceInfoKHR) {
90  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
91  .codedOffset = (VkOffset2D){ 0, 0 },
92  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
93  .baseArrayLayer = dec->layered_dpb ? dpb_slot_index : 0,
94  .imageViewBinding = vkpic->img_view_ref,
95  };
96 
97  *ref_slot = (VkVideoReferenceSlotInfoKHR) {
98  .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
99  .pNext = vkh264_ref,
100  .slotIndex = dpb_slot_index,
101  .pPictureResource = ref,
102  };
103 
104  if (ref_src)
105  *ref_src = pic;
106 
107  return 0;
108 }
109 
110 static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
111 {
112  switch (level_idc) {
113  case 10: return STD_VIDEO_H264_LEVEL_IDC_1_0;
114  case 11: return STD_VIDEO_H264_LEVEL_IDC_1_1;
115  case 12: return STD_VIDEO_H264_LEVEL_IDC_1_2;
116  case 13: return STD_VIDEO_H264_LEVEL_IDC_1_3;
117  case 20: return STD_VIDEO_H264_LEVEL_IDC_2_0;
118  case 21: return STD_VIDEO_H264_LEVEL_IDC_2_1;
119  case 22: return STD_VIDEO_H264_LEVEL_IDC_2_2;
120  case 30: return STD_VIDEO_H264_LEVEL_IDC_3_0;
121  case 31: return STD_VIDEO_H264_LEVEL_IDC_3_1;
122  case 32: return STD_VIDEO_H264_LEVEL_IDC_3_2;
123  case 40: return STD_VIDEO_H264_LEVEL_IDC_4_0;
124  case 41: return STD_VIDEO_H264_LEVEL_IDC_4_1;
125  case 42: return STD_VIDEO_H264_LEVEL_IDC_4_2;
126  case 50: return STD_VIDEO_H264_LEVEL_IDC_5_0;
127  case 51: return STD_VIDEO_H264_LEVEL_IDC_5_1;
128  case 52: return STD_VIDEO_H264_LEVEL_IDC_5_2;
129  case 60: return STD_VIDEO_H264_LEVEL_IDC_6_0;
130  case 61: return STD_VIDEO_H264_LEVEL_IDC_6_1;
131  default:
132  case 62: return STD_VIDEO_H264_LEVEL_IDC_6_2;
133  }
134 }
135 
136 static void set_sps(const SPS *sps,
137  StdVideoH264ScalingLists *vksps_scaling,
138  StdVideoH264HrdParameters *vksps_vui_header,
139  StdVideoH264SequenceParameterSetVui *vksps_vui,
140  StdVideoH264SequenceParameterSet *vksps)
141 {
142  *vksps_scaling = (StdVideoH264ScalingLists) {
143  .scaling_list_present_mask = sps->scaling_matrix_present_mask,
144  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
145  };
146 
147  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
148  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
149  vksps_scaling->ScalingList4x4[i][j] = sps->scaling_matrix4[i][ff_zigzag_scan[j]];
150 
151  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
152  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
153  vksps_scaling->ScalingList8x8[i][j] =
154  sps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
155 
156  *vksps_vui_header = (StdVideoH264HrdParameters) {
157  .cpb_cnt_minus1 = sps->cpb_cnt - 1,
158  .bit_rate_scale = sps->bit_rate_scale,
159  .initial_cpb_removal_delay_length_minus1 = sps->initial_cpb_removal_delay_length - 1,
160  .cpb_removal_delay_length_minus1 = sps->cpb_removal_delay_length - 1,
161  .dpb_output_delay_length_minus1 = sps->dpb_output_delay_length - 1,
162  .time_offset_length = sps->time_offset_length,
163  };
164 
165  for (int i = 0; i < sps->cpb_cnt; i++) {
166  vksps_vui_header->bit_rate_value_minus1[i] = sps->bit_rate_value[i] - 1;
167  vksps_vui_header->cpb_size_value_minus1[i] = sps->cpb_size_value[i] - 1;
168  vksps_vui_header->cbr_flag[i] = (sps->cpr_flag >> i) & 0x1;
169  }
170 
171  *vksps_vui = (StdVideoH264SequenceParameterSetVui) {
172  .aspect_ratio_idc = sps->vui.aspect_ratio_idc,
173  .sar_width = sps->vui.sar.num,
174  .sar_height = sps->vui.sar.den,
175  .video_format = sps->vui.video_format,
176  .colour_primaries = sps->vui.colour_primaries,
177  .transfer_characteristics = sps->vui.transfer_characteristics,
178  .matrix_coefficients = sps->vui.matrix_coeffs,
179  .num_units_in_tick = sps->num_units_in_tick,
180  .time_scale = sps->time_scale,
181  .pHrdParameters = vksps_vui_header,
182  .max_num_reorder_frames = sps->num_reorder_frames,
183  .max_dec_frame_buffering = sps->max_dec_frame_buffering,
184  .flags = (StdVideoH264SpsVuiFlags) {
185  .aspect_ratio_info_present_flag = sps->vui.aspect_ratio_info_present_flag,
186  .overscan_info_present_flag = sps->vui.overscan_info_present_flag,
187  .overscan_appropriate_flag = sps->vui.overscan_appropriate_flag,
188  .video_signal_type_present_flag = sps->vui.video_signal_type_present_flag,
189  .video_full_range_flag = sps->vui.video_full_range_flag,
190  .color_description_present_flag = sps->vui.colour_description_present_flag,
191  .chroma_loc_info_present_flag = sps->vui.chroma_location,
192  .timing_info_present_flag = sps->timing_info_present_flag,
193  .fixed_frame_rate_flag = sps->fixed_frame_rate_flag,
194  .bitstream_restriction_flag = sps->bitstream_restriction_flag,
195  .nal_hrd_parameters_present_flag = sps->nal_hrd_parameters_present_flag,
196  .vcl_hrd_parameters_present_flag = sps->vcl_hrd_parameters_present_flag,
197  },
198  };
199 
200  *vksps = (StdVideoH264SequenceParameterSet) {
201  .profile_idc = sps->profile_idc,
202  .level_idc = convert_to_vk_level_idc(sps->level_idc),
203  .seq_parameter_set_id = sps->sps_id,
204  .chroma_format_idc = sps->chroma_format_idc,
205  .bit_depth_luma_minus8 = sps->bit_depth_luma - 8,
206  .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8,
207  .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4,
208  .pic_order_cnt_type = sps->poc_type,
209  .log2_max_pic_order_cnt_lsb_minus4 = sps->poc_type ? 0 : sps->log2_max_poc_lsb - 4,
210  .offset_for_non_ref_pic = sps->offset_for_non_ref_pic,
211  .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field,
212  .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length,
213  .max_num_ref_frames = sps->ref_frame_count,
214  .pic_width_in_mbs_minus1 = sps->mb_width - 1,
215  .pic_height_in_map_units_minus1 = (sps->mb_height/(2 - sps->frame_mbs_only_flag)) - 1,
216  .frame_crop_left_offset = sps->crop_left,
217  .frame_crop_right_offset = sps->crop_right,
218  .frame_crop_top_offset = sps->crop_top,
219  .frame_crop_bottom_offset = sps->crop_bottom,
220  .flags = (StdVideoH264SpsFlags) {
221  .constraint_set0_flag = (sps->constraint_set_flags >> 0) & 0x1,
222  .constraint_set1_flag = (sps->constraint_set_flags >> 1) & 0x1,
223  .constraint_set2_flag = (sps->constraint_set_flags >> 2) & 0x1,
224  .constraint_set3_flag = (sps->constraint_set_flags >> 3) & 0x1,
225  .constraint_set4_flag = (sps->constraint_set_flags >> 4) & 0x1,
226  .constraint_set5_flag = (sps->constraint_set_flags >> 5) & 0x1,
227  .direct_8x8_inference_flag = sps->direct_8x8_inference_flag,
228  .mb_adaptive_frame_field_flag = sps->mb_aff,
229  .frame_mbs_only_flag = sps->frame_mbs_only_flag,
230  .delta_pic_order_always_zero_flag = sps->delta_pic_order_always_zero_flag,
231  .separate_colour_plane_flag = sps->residual_color_transform_flag,
232  .gaps_in_frame_num_value_allowed_flag = sps->gaps_in_frame_num_allowed_flag,
233  .qpprime_y_zero_transform_bypass_flag = sps->transform_bypass,
234  .frame_cropping_flag = sps->crop,
235  .seq_scaling_matrix_present_flag = sps->scaling_matrix_present,
236  .vui_parameters_present_flag = sps->vui_parameters_present_flag,
237  },
238  .pOffsetForRefFrame = sps->offset_for_ref_frame,
239  .pScalingLists = vksps_scaling,
240  .pSequenceParameterSetVui = vksps_vui,
241  };
242 }
243 
244 static void set_pps(const PPS *pps, const SPS *sps,
245  StdVideoH264ScalingLists *vkpps_scaling,
246  StdVideoH264PictureParameterSet *vkpps)
247 {
248  *vkpps_scaling = (StdVideoH264ScalingLists) {
249  .scaling_list_present_mask = pps->pic_scaling_matrix_present_mask,
250  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
251  };
252 
253  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
254  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
255  vkpps_scaling->ScalingList4x4[i][j] = pps->scaling_matrix4[i][ff_zigzag_scan[j]];
256 
257  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
258  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
259  vkpps_scaling->ScalingList8x8[i][j] =
260  pps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
261 
262  *vkpps = (StdVideoH264PictureParameterSet) {
263  .seq_parameter_set_id = pps->sps_id,
264  .pic_parameter_set_id = pps->pps_id,
265  .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1,
266  .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1,
267  .weighted_bipred_idc = pps->weighted_bipred_idc,
268  .pic_init_qp_minus26 = pps->init_qp - 26,
269  .pic_init_qs_minus26 = pps->init_qs - 26,
270  .chroma_qp_index_offset = pps->chroma_qp_index_offset[0],
271  .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1],
272  .flags = (StdVideoH264PpsFlags) {
273  .transform_8x8_mode_flag = pps->transform_8x8_mode,
274  .redundant_pic_cnt_present_flag = pps->redundant_pic_cnt_present,
275  .constrained_intra_pred_flag = pps->constrained_intra_pred,
276  .deblocking_filter_control_present_flag = pps->deblocking_filter_parameters_present,
277  .weighted_pred_flag = pps->weighted_pred,
278  .bottom_field_pic_order_in_frame_present_flag = pps->pic_order_present,
279  .entropy_coding_mode_flag = pps->cabac,
280  .pic_scaling_matrix_present_flag = pps->pic_scaling_matrix_present_flag,
281  },
282  .pScalingLists = vkpps_scaling,
283  };
284 }
285 
287 {
288  int err;
291  const H264Context *h = avctx->priv_data;
292 
293  /* SPS */
294  StdVideoH264ScalingLists vksps_scaling[MAX_SPS_COUNT];
295  StdVideoH264HrdParameters vksps_vui_header[MAX_SPS_COUNT];
296  StdVideoH264SequenceParameterSetVui vksps_vui[MAX_SPS_COUNT];
297  StdVideoH264SequenceParameterSet vksps[MAX_SPS_COUNT];
298 
299  /* PPS */
300  StdVideoH264ScalingLists vkpps_scaling[MAX_PPS_COUNT];
301  StdVideoH264PictureParameterSet vkpps[MAX_PPS_COUNT];
302 
303  VkVideoDecodeH264SessionParametersAddInfoKHR h264_params_info = {
304  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_ADD_INFO_KHR,
305  .pStdSPSs = vksps,
306  .stdSPSCount = 0,
307  .pStdPPSs = vkpps,
308  .stdPPSCount = 0,
309  };
310  VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
311  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
312  .pParametersAddInfo = &h264_params_info,
313  };
314  VkVideoSessionParametersCreateInfoKHR session_params_create = {
315  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
316  .pNext = &h264_params,
317  .videoSession = ctx->common.session,
318  .videoSessionParametersTemplate = VK_NULL_HANDLE,
319  };
320 
321  /* SPS list */
322  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.sps_list); i++) {
323  if (h->ps.sps_list[i]) {
324  const SPS *sps_l = h->ps.sps_list[i];
325  int idx = h264_params_info.stdSPSCount;
326  set_sps(sps_l, &vksps_scaling[idx], &vksps_vui_header[idx], &vksps_vui[idx], &vksps[idx]);
327  h264_params_info.stdSPSCount++;
328  }
329  }
330 
331  /* PPS list */
332  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.pps_list); i++) {
333  if (h->ps.pps_list[i]) {
334  const PPS *pps_l = h->ps.pps_list[i];
335  int idx = h264_params_info.stdPPSCount;
336  set_pps(pps_l, pps_l->sps, &vkpps_scaling[idx], &vkpps[idx]);
337  h264_params_info.stdPPSCount++;
338  }
339  }
340 
341  h264_params.maxStdSPSCount = h264_params_info.stdSPSCount;
342  h264_params.maxStdPPSCount = h264_params_info.stdPPSCount;
343 
344  err = ff_vk_decode_create_params(buf, avctx, ctx, &session_params_create);
345  if (err < 0)
346  return err;
347 
348  av_log(avctx, AV_LOG_DEBUG, "Created frame parameters: %i SPS %i PPS\n",
349  h264_params_info.stdSPSCount, h264_params_info.stdPPSCount);
350 
351  return 0;
352 }
353 
355  av_unused const uint8_t *buffer,
356  av_unused uint32_t size)
357 {
358  int err;
359  int dpb_slot_index = 0;
360  H264Context *h = avctx->priv_data;
361  H264Picture *pic = h->cur_pic_ptr;
364  FFVulkanDecodePicture *vp = &hp->vp;
365 
366  if (!dec->session_params) {
367  err = vk_h264_create_params(avctx, &dec->session_params);
368  if (err < 0)
369  return err;
370  }
371 
372  /* Fill in main slot */
373  dpb_slot_index = 0;
374  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
375  if (pic == &h->DPB[slot]) {
376  dpb_slot_index = slot;
377  break;
378  }
379  }
380 
381  err = vk_h264_fill_pict(avctx, NULL, &vp->ref_slot, &vp->ref,
382  &hp->vkh264_ref, &hp->h264_ref, pic, 1,
383  h->DPB[dpb_slot_index].field_picture,
384  h->DPB[dpb_slot_index].reference,
385  dpb_slot_index);
386  if (err < 0)
387  return err;
388 
389  /* Fill in short-term references */
390  for (int i = 0; i < h->short_ref_count; i++) {
391  dpb_slot_index = 0;
392  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
393  if (h->short_ref[i] == &h->DPB[slot]) {
394  dpb_slot_index = slot;
395  break;
396  }
397  }
398  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
399  &vp->refs[i], &hp->vkh264_refs[i],
400  &hp->h264_refs[i], h->short_ref[i], 0,
401  h->DPB[dpb_slot_index].field_picture,
402  h->DPB[dpb_slot_index].reference,
403  dpb_slot_index);
404  if (err < 0)
405  return err;
406  }
407 
408  /* Fill in long-term refs */
409  for (int r = 0, i = h->short_ref_count; r < H264_MAX_DPB_FRAMES &&
410  i < h->short_ref_count + h->long_ref_count; r++) {
411  if (!h->long_ref[r])
412  continue;
413 
414  dpb_slot_index = 0;
415  for (unsigned slot = 0; slot < 16; slot++) {
416  if (h->long_ref[r] == &h->DPB[slot]) {
417  dpb_slot_index = slot;
418  break;
419  }
420  }
421  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
422  &vp->refs[i], &hp->vkh264_refs[i],
423  &hp->h264_refs[i], h->long_ref[r], 0,
424  h->DPB[dpb_slot_index].field_picture,
425  h->DPB[dpb_slot_index].reference,
426  dpb_slot_index);
427  if (err < 0)
428  return err;
429  i++;
430  }
431 
432  hp->h264pic = (StdVideoDecodeH264PictureInfo) {
433  .seq_parameter_set_id = pic->pps->sps_id,
434  .pic_parameter_set_id = pic->pps->pps_id,
435  .frame_num = 0, /* Set later */
436  .idr_pic_id = 0, /* Set later */
437  .PicOrderCnt[0] = pic->field_poc[0],
438  .PicOrderCnt[1] = pic->field_poc[1],
439  .flags = (StdVideoDecodeH264PictureInfoFlags) {
440  .field_pic_flag = FIELD_PICTURE(h),
441  .is_intra = 1, /* Set later */
442  .IdrPicFlag = h->picture_idr,
443  .bottom_field_flag = h->picture_structure != PICT_FRAME &&
444  h->picture_structure & PICT_BOTTOM_FIELD,
445  .is_reference = h->nal_ref_idc != 0,
446  .complementary_field_pair = h->first_field && FIELD_PICTURE(h),
447  },
448  };
449 
450  hp->h264_pic_info = (VkVideoDecodeH264PictureInfoKHR) {
451  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PICTURE_INFO_KHR,
452  .pStdPictureInfo = &hp->h264pic,
453  };
454 
455  vp->decode_info = (VkVideoDecodeInfoKHR) {
456  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_INFO_KHR,
457  .pNext = &hp->h264_pic_info,
458  .flags = 0x0,
459  .pSetupReferenceSlot = &vp->ref_slot,
460  .referenceSlotCount = h->short_ref_count + h->long_ref_count,
461  .pReferenceSlots = vp->ref_slots,
462  .dstPictureResource = (VkVideoPictureResourceInfoKHR) {
463  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
464  .codedOffset = (VkOffset2D){ 0, 0 },
465  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
466  .baseArrayLayer = 0,
467  .imageViewBinding = vp->img_view_out,
468  },
469  };
470 
471  return 0;
472 }
473 
475  const uint8_t *data,
476  uint32_t size)
477 {
478  const H264Context *h = avctx->priv_data;
479  const H264SliceContext *sl = &h->slice_ctx[0];
480  H264VulkanDecodePicture *hp = h->cur_pic_ptr->hwaccel_picture_private;
481  FFVulkanDecodePicture *vp = &hp->vp;
482 
483  int err = ff_vk_decode_add_slice(avctx, vp, data, size, 1,
484  &hp->h264_pic_info.sliceCount,
485  &hp->h264_pic_info.pSliceOffsets);
486  if (err < 0)
487  return err;
488 
489  hp->h264pic.frame_num = sl->frame_num;
490  hp->h264pic.idr_pic_id = sl->idr_pic_id;
491 
492  /* Frame is only intra of all slices are marked as intra */
494  hp->h264pic.flags.is_intra = 0;
495 
496  return 0;
497 }
498 
500 {
501  const H264Context *h = avctx->priv_data;
502  H264Picture *pic = h->cur_pic_ptr;
505  FFVulkanDecodePicture *vp = &hp->vp;
507  AVFrame *rav[H264_MAX_PICTURE_COUNT] = { 0 };
508 
509  if (!hp->h264_pic_info.sliceCount)
510  return 0;
511 
512  if (!vp->slices_buf)
513  return AVERROR(EINVAL);
514 
515  if (!dec->session_params) {
516  int err = vk_h264_create_params(avctx, &dec->session_params);
517  if (err < 0)
518  return err;
519 
520  hp->h264pic.seq_parameter_set_id = pic->pps->sps_id;
521  hp->h264pic.pic_parameter_set_id = pic->pps->pps_id;
522  }
523 
524  for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
525  H264Picture *rp = hp->ref_src[i];
527 
528  rvp[i] = &rhp->vp;
529  rav[i] = hp->ref_src[i]->f;
530  }
531 
532  av_log(avctx, AV_LOG_VERBOSE, "Decoding frame, %"SIZE_SPECIFIER" bytes, %i slices\n",
533  vp->slices_size, hp->h264_pic_info.sliceCount);
534 
535  return ff_vk_decode_frame(avctx, pic->f, vp, rav, rvp);
536 }
537 
539 {
540  AVHWDeviceContext *hwctx = _hwctx.nc;
542 
543  /* Free frame resources, this also destroys the session parameters. */
544  ff_vk_decode_free_frame(hwctx, &hp->vp);
545 }
546 
548  .p.name = "h264_vulkan",
549  .p.type = AVMEDIA_TYPE_VIDEO,
550  .p.id = AV_CODEC_ID_H264,
551  .p.pix_fmt = AV_PIX_FMT_VULKAN,
552  .start_frame = &vk_h264_start_frame,
553  .decode_slice = &vk_h264_decode_slice,
554  .end_frame = &vk_h264_end_frame,
555  .free_frame_priv = &vk_h264_free_frame_priv,
556  .frame_priv_data_size = sizeof(H264VulkanDecodePicture),
559  .decode_params = &ff_vk_params_invalidate,
562  .frame_params = &ff_vk_frame_params,
563  .priv_data_size = sizeof(FFVulkanDecodeContext),
565 };
PICT_FRAME
#define PICT_FRAME
Definition: mpegutils.h:38
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:99
r
const char * r
Definition: vf_curves.c:126
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
H264Picture::f
AVFrame * f
Definition: h264dec.h:107
H264VulkanDecodePicture::h264_ref
StdVideoDecodeH264ReferenceInfo h264_ref
Definition: vulkan_h264.c:33
set_sps
static void set_sps(const SPS *sps, StdVideoH264ScalingLists *vksps_scaling, StdVideoH264HrdParameters *vksps_vui_header, StdVideoH264SequenceParameterSetVui *vksps_vui, StdVideoH264SequenceParameterSet *vksps)
Definition: vulkan_h264.c:136
FFVulkanDecodeContext::shared_ctx
FFVulkanDecodeShared * shared_ctx
Definition: vulkan_decode.h:57
H264VulkanDecodePicture::h264_pic_info
VkVideoDecodeH264PictureInfoKHR h264_pic_info
Definition: vulkan_h264.c:43
av_unused
#define av_unused
Definition: attributes.h:131
H264Picture::pps
const PPS * pps
Definition: h264dec.h:150
FFHWAccel::p
AVHWAccel p
The public AVHWAccel.
Definition: hwaccel_internal.h:38
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
AVFrame::width
int width
Definition: frame.h:412
level_idc
int level_idc
Definition: h264_levels.c:29
set_pps
static void set_pps(const PPS *pps, const SPS *sps, StdVideoH264ScalingLists *vkpps_scaling, StdVideoH264PictureParameterSet *vkpps)
Definition: vulkan_h264.c:244
data
const char data[16]
Definition: mxf.c:148
H264_MAX_DPB_FRAMES
@ H264_MAX_DPB_FRAMES
Definition: h264.h:76
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
PICT_BOTTOM_FIELD
#define PICT_BOTTOM_FIELD
Definition: mpegutils.h:37
H264VulkanDecodePicture::vkh264_ref
VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref
Definition: vulkan_h264.c:34
H264VulkanDecodePicture::h264pic
StdVideoDecodeH264PictureInfo h264pic
Definition: vulkan_h264.c:42
FFRefStructOpaque
RefStruct is an API for creating reference-counted objects with minimal overhead.
Definition: refstruct.h:58
vk_h264_create_params
static int vk_h264_create_params(AVCodecContext *avctx, AVBufferRef **buf)
Definition: vulkan_h264.c:286
H264Picture::invalid_gap
int invalid_gap
Definition: h264dec.h:146
FFVulkanDecodeContext
Definition: vulkan_decode.h:56
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:383
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:175
FFVulkanDecodePicture::ref
VkVideoPictureResourceInfoKHR ref
Definition: vulkan_decode.h:87
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:58
H264Picture::frame_num
int frame_num
frame_num (raw frame_num from slice header)
Definition: h264dec.h:128
H264SliceContext
Definition: h264dec.h:172
ff_h264_vulkan_hwaccel
const FFHWAccel ff_h264_vulkan_hwaccel
Definition: vulkan_h264.c:547
FFHWAccel
Definition: hwaccel_internal.h:34
HWACCEL_CAP_THREAD_SAFE
#define HWACCEL_CAP_THREAD_SAFE
Definition: hwaccel_internal.h:32
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:331
FFVulkanDecodeShared
Definition: vulkan_decode.h:37
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:62
vk_h264_free_frame_priv
static void vk_h264_free_frame_priv(FFRefStructOpaque _hwctx, void *data)
Definition: vulkan_h264.c:538
FFVulkanDecodePicture::img_view_out
VkImageView img_view_out
Definition: vulkan_decode.h:78
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
vk_h264_decode_slice
static int vk_h264_decode_slice(AVCodecContext *avctx, const uint8_t *data, uint32_t size)
Definition: vulkan_h264.c:474
FFVulkanDecodePicture::refs
VkVideoPictureResourceInfoKHR refs[36]
Definition: vulkan_decode.h:91
H264VulkanDecodePicture
Definition: vulkan_h264.c:29
FFVulkanDecodePicture
Definition: vulkan_decode.h:74
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1061
PPS::pps_id
unsigned int pps_id
Definition: h264_ps.h:111
H264VulkanDecodePicture::h264_refs
StdVideoDecodeH264ReferenceInfo h264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:38
FIELD_PICTURE
#define FIELD_PICTURE(h)
Definition: h264dec.h:67
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
PICT_TOP_FIELD
#define PICT_TOP_FIELD
Definition: mpegutils.h:36
H264VulkanDecodePicture::ref_src
H264Picture * ref_src[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:37
H264SliceContext::slice_type
int slice_type
Definition: h264dec.h:178
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
HWACCEL_CAP_ASYNC_SAFE
#define HWACCEL_CAP_ASYNC_SAFE
Header providing the internals of AVHWAccel.
Definition: hwaccel_internal.h:31
FFVulkanDecodePicture::img_view_ref
VkImageView img_view_ref
Definition: vulkan_decode.h:77
NULL
#define NULL
Definition: coverity.c:32
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:561
convert_to_vk_level_idc
static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
Definition: vulkan_h264.c:110
SPS
Sequence parameter set.
Definition: h264_ps.h:44
AVCodecContext::internal
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:480
AV_PICTURE_TYPE_SI
@ AV_PICTURE_TYPE_SI
Switching Intra.
Definition: avutil.h:283
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:279
PPS
Picture parameter set.
Definition: h264_ps.h:110
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1090
flush
void(* flush)(AVBSFContext *ctx)
Definition: dts2pts.c:367
MAX_PPS_COUNT
#define MAX_PPS_COUNT
Definition: h264_ps.h:38
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:1004
h264_ps.h
H264Picture::pic_id
int pic_id
pic_num (short -> no wrap version of pic_num, pic_num & max_pic_num; long -> long_pic_num)
Definition: h264dec.h:131
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:365
ff_zigzag_scan
const uint8_t ff_zigzag_scan[16+1]
Definition: mathtables.c:109
H264Picture::reference
int reference
Definition: h264dec.h:144
pps
static int FUNC() pps(CodedBitstreamContext *ctx, RWContext *rw, H264RawPPS *current)
Definition: cbs_h264_syntax_template.c:404
AVCodecInternal::hwaccel_priv_data
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:121
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
ff_vk_dec_h264_ext
const VkExtensionProperties ff_vk_dec_h264_ext
Definition: vulkan_h264.c:24
size
int size
Definition: twinvq_data.h:10344
h264_scaling_list8_order
const static int h264_scaling_list8_order[]
Definition: vulkan_h264.c:46
FFRefStructOpaque::nc
void * nc
Definition: refstruct.h:59
AVHWAccel::name
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:2139
h264dec.h
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
H264Context
H264Context.
Definition: h264dec.h:332
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:245
vk_h264_end_frame
static int vk_h264_end_frame(AVCodecContext *avctx)
Definition: vulkan_h264.c:499
H264SliceContext::frame_num
int frame_num
Definition: h264dec.h:320
H264VulkanDecodePicture::vp
FFVulkanDecodePicture vp
Definition: vulkan_h264.c:30
vk_h264_start_frame
static int vk_h264_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
Definition: vulkan_h264.c:354
FFVulkanDecodePicture::ref_slot
VkVideoReferenceSlotInfoKHR ref_slot
Definition: vulkan_decode.h:88
update_thread_context
the pkt_dts and pkt_pts fields in AVFrame will work as usual Restrictions on codec whose streams don t reset across will not work because their bitstreams cannot be decoded in parallel *The contents of buffers must not be read before as well as code calling up to before the decode process starts Call have update_thread_context() run it in the next thread. Add AV_CODEC_CAP_FRAME_THREADS to the codec capabilities. There will be very little speed gain at this point but it should work. If there are inter-frame dependencies
FFVulkanDecodePicture::ref_slots
VkVideoReferenceSlotInfoKHR ref_slots[36]
Definition: vulkan_decode.h:92
ff_zigzag_direct
const uint8_t ff_zigzag_direct[64]
Definition: mathtables.c:98
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:301
PPS::sps
const SPS * sps
RefStruct reference.
Definition: h264_ps.h:141
vk_h264_fill_pict
static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src, VkVideoReferenceSlotInfoKHR *ref_slot, VkVideoPictureResourceInfoKHR *ref, VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, StdVideoDecodeH264ReferenceInfo *h264_ref, H264Picture *pic, int is_current, int is_field, int picture_structure, int dpb_slot_index)
Definition: vulkan_h264.c:48
sps
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
Definition: cbs_h264_syntax_template.c:260
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:239
SIZE_SPECIFIER
#define SIZE_SPECIFIER
Definition: internal.h:141
PPS::sps_id
unsigned int sps_id
Definition: h264_ps.h:112
AVCodecContext
main external API structure.
Definition: avcodec.h:445
AVFrame::height
int height
Definition: frame.h:412
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:61
H264Picture::field_poc
int field_poc[2]
top/bottom POC
Definition: h264dec.h:126
buffer
the frame and frame reference mechanism is intended to as much as expensive copies of that data while still allowing the filters to produce correct results The data is stored in buffers represented by AVFrame structures Several references can point to the same frame buffer
Definition: filter_design.txt:49
H264Picture
Definition: h264dec.h:106
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:109
MAX_SPS_COUNT
#define MAX_SPS_COUNT
Definition: h264_ps.h:37
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:68
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:98
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
H264Picture::hwaccel_picture_private
void * hwaccel_picture_private
RefStruct reference for hardware accelerator private data.
Definition: h264dec.h:122
vulkan_decode.h
H264SliceContext::idr_pic_id
int idr_pic_id
Definition: h264dec.h:321
AVCodecContext::priv_data
void * priv_data
Definition: avcodec.h:472
FFVulkanDecodeContext::layered_dpb
int layered_dpb
Definition: vulkan_decode.h:62
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1105
h
h
Definition: vp9dsp_template.c:2038
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:95
H264_MAX_PICTURE_COUNT
#define H264_MAX_PICTURE_COUNT
Definition: h264dec.h:49
H264VulkanDecodePicture::vkh264_refs
VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:39
H264Picture::long_ref
int long_ref
1->long term reference 0->short term reference
Definition: h264dec.h:133