FFmpeg
vulkan_h264.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #include "h264dec.h"
20 #include "h264_ps.h"
21 
22 #include "vulkan_decode.h"
23 
24 const VkExtensionProperties ff_vk_dec_h264_ext = {
25  .extensionName = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_EXTENSION_NAME,
26  .specVersion = VK_STD_VULKAN_VIDEO_CODEC_H264_DECODE_SPEC_VERSION,
27 };
28 
29 typedef struct H264VulkanDecodePicture {
31 
32  /* Current picture */
33  StdVideoDecodeH264ReferenceInfo h264_ref;
34  VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref;
35 
36  /* Picture refs */
38  StdVideoDecodeH264ReferenceInfo h264_refs [H264_MAX_PICTURE_COUNT];
39  VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT];
40 
41  /* Current picture (contd.) */
42  StdVideoDecodeH264PictureInfo h264pic;
43  VkVideoDecodeH264PictureInfoKHR h264_pic_info;
45 
46 const static int h264_scaling_list8_order[] = { 0, 3, 1, 4, 2, 5 };
47 
48 static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src,
49  VkVideoReferenceSlotInfoKHR *ref_slot, /* Main structure */
50  VkVideoPictureResourceInfoKHR *ref, /* Goes in ^ */
51  VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, /* Goes in ^ */
52  StdVideoDecodeH264ReferenceInfo *h264_ref, /* Goes in ^ */
53  H264Picture *pic, int is_current,
54  int is_field, int picture_structure,
55  int dpb_slot_index)
56 {
59  FFVulkanDecodePicture *vkpic = &hp->vp;
60 
61  int err = ff_vk_decode_prepare_frame(dec, pic->f, vkpic, is_current,
62  dec->dedicated_dpb);
63  if (err < 0)
64  return err;
65 
66  *h264_ref = (StdVideoDecodeH264ReferenceInfo) {
67  .FrameNum = pic->long_ref ? pic->pic_id : pic->frame_num,
68  .PicOrderCnt = { pic->field_poc[0], pic->field_poc[1] },
69  .flags = (StdVideoDecodeH264ReferenceInfoFlags) {
70  .top_field_flag = is_field ? !!(picture_structure & PICT_TOP_FIELD) : 0,
71  .bottom_field_flag = is_field ? !!(picture_structure & PICT_BOTTOM_FIELD) : 0,
72  .used_for_long_term_reference = pic->reference && pic->long_ref,
73  /*
74  * flags.is_non_existing is used to indicate whether the picture is marked as
75  * “non-existing” as defined in section 8.2.5.2 of the ITU-T H.264 Specification;
76  * 8.2.5.2 Decoding process for gaps in frame_num
77  * corresponds to the code in h264_slice.c:h264_field_start,
78  * which sets the invalid_gap flag when decoding.
79  */
80  .is_non_existing = pic->invalid_gap,
81  },
82  };
83 
84  *vkh264_ref = (VkVideoDecodeH264DpbSlotInfoKHR) {
85  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_DPB_SLOT_INFO_KHR,
86  .pStdReferenceInfo = h264_ref,
87  };
88 
89  *ref = (VkVideoPictureResourceInfoKHR) {
90  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
91  .codedOffset = (VkOffset2D){ 0, 0 },
92  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
93  .baseArrayLayer = dec->layered_dpb ? dpb_slot_index : 0,
94  .imageViewBinding = vkpic->img_view_ref,
95  };
96 
97  *ref_slot = (VkVideoReferenceSlotInfoKHR) {
98  .sType = VK_STRUCTURE_TYPE_VIDEO_REFERENCE_SLOT_INFO_KHR,
99  .pNext = vkh264_ref,
100  .slotIndex = dpb_slot_index,
101  .pPictureResource = ref,
102  };
103 
104  if (ref_src)
105  *ref_src = pic;
106 
107  return 0;
108 }
109 
110 static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
111 {
112  switch (level_idc) {
113  case 10: return STD_VIDEO_H264_LEVEL_IDC_1_0;
114  case 11: return STD_VIDEO_H264_LEVEL_IDC_1_1;
115  case 12: return STD_VIDEO_H264_LEVEL_IDC_1_2;
116  case 13: return STD_VIDEO_H264_LEVEL_IDC_1_3;
117  case 20: return STD_VIDEO_H264_LEVEL_IDC_2_0;
118  case 21: return STD_VIDEO_H264_LEVEL_IDC_2_1;
119  case 22: return STD_VIDEO_H264_LEVEL_IDC_2_2;
120  case 30: return STD_VIDEO_H264_LEVEL_IDC_3_0;
121  case 31: return STD_VIDEO_H264_LEVEL_IDC_3_1;
122  case 32: return STD_VIDEO_H264_LEVEL_IDC_3_2;
123  case 40: return STD_VIDEO_H264_LEVEL_IDC_4_0;
124  case 41: return STD_VIDEO_H264_LEVEL_IDC_4_1;
125  case 42: return STD_VIDEO_H264_LEVEL_IDC_4_2;
126  case 50: return STD_VIDEO_H264_LEVEL_IDC_5_0;
127  case 51: return STD_VIDEO_H264_LEVEL_IDC_5_1;
128  case 52: return STD_VIDEO_H264_LEVEL_IDC_5_2;
129  case 60: return STD_VIDEO_H264_LEVEL_IDC_6_0;
130  case 61: return STD_VIDEO_H264_LEVEL_IDC_6_1;
131  default:
132  case 62: return STD_VIDEO_H264_LEVEL_IDC_6_2;
133  }
134 }
135 
136 static void set_sps(const SPS *sps,
137  StdVideoH264ScalingLists *vksps_scaling,
138  StdVideoH264HrdParameters *vksps_vui_header,
139  StdVideoH264SequenceParameterSetVui *vksps_vui,
140  StdVideoH264SequenceParameterSet *vksps)
141 {
142  *vksps_scaling = (StdVideoH264ScalingLists) {
143  .scaling_list_present_mask = sps->scaling_matrix_present_mask,
144  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
145  };
146 
147  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
148  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
149  vksps_scaling->ScalingList4x4[i][j] = sps->scaling_matrix4[i][ff_zigzag_scan[j]];
150 
151  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
152  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
153  vksps_scaling->ScalingList8x8[i][j] =
154  sps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
155 
156  *vksps_vui_header = (StdVideoH264HrdParameters) {
157  .cpb_cnt_minus1 = sps->cpb_cnt - 1,
158  .bit_rate_scale = sps->bit_rate_scale,
159  .initial_cpb_removal_delay_length_minus1 = sps->initial_cpb_removal_delay_length - 1,
160  .cpb_removal_delay_length_minus1 = sps->cpb_removal_delay_length - 1,
161  .dpb_output_delay_length_minus1 = sps->dpb_output_delay_length - 1,
162  .time_offset_length = sps->time_offset_length,
163  };
164 
165  for (int i = 0; i < sps->cpb_cnt; i++) {
166  vksps_vui_header->bit_rate_value_minus1[i] = sps->bit_rate_value[i] - 1;
167  vksps_vui_header->cpb_size_value_minus1[i] = sps->cpb_size_value[i] - 1;
168  vksps_vui_header->cbr_flag[i] = (sps->cpr_flag >> i) & 0x1;
169  }
170 
171  *vksps_vui = (StdVideoH264SequenceParameterSetVui) {
172  .aspect_ratio_idc = sps->vui.aspect_ratio_idc,
173  .sar_width = sps->vui.sar.num,
174  .sar_height = sps->vui.sar.den,
175  .video_format = sps->vui.video_format,
176  .colour_primaries = sps->vui.colour_primaries,
177  .transfer_characteristics = sps->vui.transfer_characteristics,
178  .matrix_coefficients = sps->vui.matrix_coeffs,
179  .num_units_in_tick = sps->num_units_in_tick,
180  .time_scale = sps->time_scale,
181  .pHrdParameters = vksps_vui_header,
182  .max_num_reorder_frames = sps->num_reorder_frames,
183  .max_dec_frame_buffering = sps->max_dec_frame_buffering,
184  .flags = (StdVideoH264SpsVuiFlags) {
185  .aspect_ratio_info_present_flag = sps->vui.aspect_ratio_info_present_flag,
186  .overscan_info_present_flag = sps->vui.overscan_info_present_flag,
187  .overscan_appropriate_flag = sps->vui.overscan_appropriate_flag,
188  .video_signal_type_present_flag = sps->vui.video_signal_type_present_flag,
189  .video_full_range_flag = sps->vui.video_full_range_flag,
190  .color_description_present_flag = sps->vui.colour_description_present_flag,
191  .chroma_loc_info_present_flag = sps->vui.chroma_location,
192  .timing_info_present_flag = sps->timing_info_present_flag,
193  .fixed_frame_rate_flag = sps->fixed_frame_rate_flag,
194  .bitstream_restriction_flag = sps->bitstream_restriction_flag,
195  .nal_hrd_parameters_present_flag = sps->nal_hrd_parameters_present_flag,
196  .vcl_hrd_parameters_present_flag = sps->vcl_hrd_parameters_present_flag,
197  },
198  };
199 
200  *vksps = (StdVideoH264SequenceParameterSet) {
201  .profile_idc = sps->profile_idc,
202  .level_idc = convert_to_vk_level_idc(sps->level_idc),
203  .seq_parameter_set_id = sps->sps_id,
204  .chroma_format_idc = sps->chroma_format_idc,
205  .bit_depth_luma_minus8 = sps->bit_depth_luma - 8,
206  .bit_depth_chroma_minus8 = sps->bit_depth_chroma - 8,
207  .log2_max_frame_num_minus4 = sps->log2_max_frame_num - 4,
208  .pic_order_cnt_type = sps->poc_type,
209  .log2_max_pic_order_cnt_lsb_minus4 = sps->poc_type ? 0 : sps->log2_max_poc_lsb - 4,
210  .offset_for_non_ref_pic = sps->offset_for_non_ref_pic,
211  .offset_for_top_to_bottom_field = sps->offset_for_top_to_bottom_field,
212  .num_ref_frames_in_pic_order_cnt_cycle = sps->poc_cycle_length,
213  .max_num_ref_frames = sps->ref_frame_count,
214  .pic_width_in_mbs_minus1 = sps->mb_width - 1,
215  .pic_height_in_map_units_minus1 = (sps->mb_height/(2 - sps->frame_mbs_only_flag)) - 1,
216  .frame_crop_left_offset = sps->crop_left,
217  .frame_crop_right_offset = sps->crop_right,
218  .frame_crop_top_offset = sps->crop_top,
219  .frame_crop_bottom_offset = sps->crop_bottom,
220  .flags = (StdVideoH264SpsFlags) {
221  .constraint_set0_flag = (sps->constraint_set_flags >> 0) & 0x1,
222  .constraint_set1_flag = (sps->constraint_set_flags >> 1) & 0x1,
223  .constraint_set2_flag = (sps->constraint_set_flags >> 2) & 0x1,
224  .constraint_set3_flag = (sps->constraint_set_flags >> 3) & 0x1,
225  .constraint_set4_flag = (sps->constraint_set_flags >> 4) & 0x1,
226  .constraint_set5_flag = (sps->constraint_set_flags >> 5) & 0x1,
227  .direct_8x8_inference_flag = sps->direct_8x8_inference_flag,
228  .mb_adaptive_frame_field_flag = sps->mb_aff,
229  .frame_mbs_only_flag = sps->frame_mbs_only_flag,
230  .delta_pic_order_always_zero_flag = sps->delta_pic_order_always_zero_flag,
231  .separate_colour_plane_flag = sps->residual_color_transform_flag,
232  .gaps_in_frame_num_value_allowed_flag = sps->gaps_in_frame_num_allowed_flag,
233  .qpprime_y_zero_transform_bypass_flag = sps->transform_bypass,
234  .frame_cropping_flag = sps->crop,
235  .seq_scaling_matrix_present_flag = sps->scaling_matrix_present,
236  .vui_parameters_present_flag = sps->vui_parameters_present_flag,
237  },
238  .pOffsetForRefFrame = sps->offset_for_ref_frame,
239  .pScalingLists = vksps_scaling,
240  .pSequenceParameterSetVui = vksps_vui,
241  };
242 }
243 
244 static void set_pps(const PPS *pps, const SPS *sps,
245  StdVideoH264ScalingLists *vkpps_scaling,
246  StdVideoH264PictureParameterSet *vkpps)
247 {
248  *vkpps_scaling = (StdVideoH264ScalingLists) {
249  .scaling_list_present_mask = pps->pic_scaling_matrix_present_mask,
250  .use_default_scaling_matrix_mask = 0, /* We already fill in the default matrix */
251  };
252 
253  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_LISTS; i++)
254  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_4X4_NUM_ELEMENTS; j++)
255  vkpps_scaling->ScalingList4x4[i][j] = pps->scaling_matrix4[i][ff_zigzag_scan[j]];
256 
257  for (int i = 0; i < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_LISTS; i++)
258  for (int j = 0; j < STD_VIDEO_H264_SCALING_LIST_8X8_NUM_ELEMENTS; j++)
259  vkpps_scaling->ScalingList8x8[i][j] =
260  pps->scaling_matrix8[h264_scaling_list8_order[i]][ff_zigzag_direct[j]];
261 
262  *vkpps = (StdVideoH264PictureParameterSet) {
263  .seq_parameter_set_id = pps->sps_id,
264  .pic_parameter_set_id = pps->pps_id,
265  .num_ref_idx_l0_default_active_minus1 = pps->ref_count[0] - 1,
266  .num_ref_idx_l1_default_active_minus1 = pps->ref_count[1] - 1,
267  .weighted_bipred_idc = pps->weighted_bipred_idc,
268  .pic_init_qp_minus26 = pps->init_qp - 26,
269  .pic_init_qs_minus26 = pps->init_qs - 26,
270  .chroma_qp_index_offset = pps->chroma_qp_index_offset[0],
271  .second_chroma_qp_index_offset = pps->chroma_qp_index_offset[1],
272  .flags = (StdVideoH264PpsFlags) {
273  .transform_8x8_mode_flag = pps->transform_8x8_mode,
274  .redundant_pic_cnt_present_flag = pps->redundant_pic_cnt_present,
275  .constrained_intra_pred_flag = pps->constrained_intra_pred,
276  .deblocking_filter_control_present_flag = pps->deblocking_filter_parameters_present,
277  .weighted_pred_flag = pps->weighted_pred,
278  .bottom_field_pic_order_in_frame_present_flag = pps->pic_order_present,
279  .entropy_coding_mode_flag = pps->cabac,
280  .pic_scaling_matrix_present_flag = pps->pic_scaling_matrix_present_flag,
281  },
282  .pScalingLists = vkpps_scaling,
283  };
284 }
285 
287 {
288  int err;
291  const H264Context *h = avctx->priv_data;
292 
293  /* SPS */
294  StdVideoH264ScalingLists vksps_scaling[MAX_SPS_COUNT];
295  StdVideoH264HrdParameters vksps_vui_header[MAX_SPS_COUNT];
296  StdVideoH264SequenceParameterSetVui vksps_vui[MAX_SPS_COUNT];
297  StdVideoH264SequenceParameterSet vksps[MAX_SPS_COUNT];
298 
299  /* PPS */
300  StdVideoH264ScalingLists vkpps_scaling[MAX_PPS_COUNT];
301  StdVideoH264PictureParameterSet vkpps[MAX_PPS_COUNT];
302 
303  VkVideoDecodeH264SessionParametersAddInfoKHR h264_params_info = {
304  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_ADD_INFO_KHR,
305  .pStdSPSs = vksps,
306  .stdSPSCount = 0,
307  .pStdPPSs = vkpps,
308  .stdPPSCount = 0,
309  };
310  VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
311  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
312  .pParametersAddInfo = &h264_params_info,
313  };
314  VkVideoSessionParametersCreateInfoKHR session_params_create = {
315  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
316  .pNext = &h264_params,
317  .videoSession = ctx->common.session,
318  .videoSessionParametersTemplate = NULL,
319  };
320 
321  /* SPS list */
322  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.sps_list); i++) {
323  if (h->ps.sps_list[i]) {
324  const SPS *sps_l = (const SPS *)h->ps.sps_list[i]->data;
325  int idx = h264_params_info.stdSPSCount;
326  set_sps(sps_l, &vksps_scaling[idx], &vksps_vui_header[idx], &vksps_vui[idx], &vksps[idx]);
327  h264_params_info.stdSPSCount++;
328  }
329  }
330 
331  /* PPS list */
332  for (int i = 0; i < FF_ARRAY_ELEMS(h->ps.pps_list); i++) {
333  if (h->ps.pps_list[i]) {
334  const PPS *pps_l = (const PPS *)h->ps.pps_list[i]->data;
335  int idx = h264_params_info.stdPPSCount;
336  set_pps(pps_l, pps_l->sps, &vkpps_scaling[idx], &vkpps[idx]);
337  h264_params_info.stdPPSCount++;
338  }
339  }
340 
341  h264_params.maxStdSPSCount = h264_params_info.stdSPSCount;
342  h264_params.maxStdPPSCount = h264_params_info.stdPPSCount;
343 
344  err = ff_vk_decode_create_params(buf, avctx, ctx, &session_params_create);
345  if (err < 0)
346  return err;
347 
348  av_log(avctx, AV_LOG_DEBUG, "Created frame parameters: %i SPS %i PPS\n",
349  h264_params_info.stdSPSCount, h264_params_info.stdPPSCount);
350 
351  return 0;
352 }
353 
355  av_unused const uint8_t *buffer,
356  av_unused uint32_t size)
357 {
358  int err;
359  int dpb_slot_index = 0;
360  H264Context *h = avctx->priv_data;
361  H264Picture *pic = h->cur_pic_ptr;
364  FFVulkanDecodePicture *vp = &hp->vp;
365 
366  if (!dec->session_params) {
367  err = vk_h264_create_params(avctx, &dec->session_params);
368  if (err < 0)
369  return err;
370  }
371 
372  /* Fill in main slot */
373  dpb_slot_index = 0;
374  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
375  if (pic == &h->DPB[slot]) {
376  dpb_slot_index = slot;
377  break;
378  }
379  }
380 
381  err = vk_h264_fill_pict(avctx, NULL, &vp->ref_slot, &vp->ref,
382  &hp->vkh264_ref, &hp->h264_ref, pic, 1,
383  h->DPB[dpb_slot_index].field_picture,
384  h->DPB[dpb_slot_index].reference,
385  dpb_slot_index);
386  if (err < 0)
387  return err;
388 
389  /* Fill in short-term references */
390  for (int i = 0; i < h->short_ref_count; i++) {
391  dpb_slot_index = 0;
392  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
393  if (h->short_ref[i] == &h->DPB[slot]) {
394  dpb_slot_index = slot;
395  break;
396  }
397  }
398  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
399  &vp->refs[i], &hp->vkh264_refs[i],
400  &hp->h264_refs[i], h->short_ref[i], 0,
401  h->DPB[dpb_slot_index].field_picture,
402  h->DPB[dpb_slot_index].reference,
403  dpb_slot_index);
404  if (err < 0)
405  return err;
406  }
407 
408  /* Fill in long-term refs */
409  for (int r = 0, i = h->short_ref_count; i < h->short_ref_count + h->long_ref_count; i++, r++) {
410  dpb_slot_index = 0;
411  for (unsigned slot = 0; slot < H264_MAX_PICTURE_COUNT; slot++) {
412  if (h->long_ref[i] == &h->DPB[slot]) {
413  dpb_slot_index = slot;
414  break;
415  }
416  }
417  err = vk_h264_fill_pict(avctx, &hp->ref_src[i], &vp->ref_slots[i],
418  &vp->refs[i], &hp->vkh264_refs[i],
419  &hp->h264_refs[i], h->long_ref[r], 0,
420  h->DPB[dpb_slot_index].field_picture,
421  h->DPB[dpb_slot_index].reference,
422  dpb_slot_index);
423  if (err < 0)
424  return err;
425  }
426 
427  hp->h264pic = (StdVideoDecodeH264PictureInfo) {
428  .seq_parameter_set_id = pic->pps->sps_id,
429  .pic_parameter_set_id = pic->pps->pps_id,
430  .frame_num = 0, /* Set later */
431  .idr_pic_id = 0, /* Set later */
432  .PicOrderCnt[0] = pic->field_poc[0],
433  .PicOrderCnt[1] = pic->field_poc[1],
434  .flags = (StdVideoDecodeH264PictureInfoFlags) {
435  .field_pic_flag = FIELD_PICTURE(h),
436  .is_intra = 1, /* Set later */
437  .IdrPicFlag = h->picture_idr,
438  .bottom_field_flag = h->picture_structure != PICT_FRAME &&
439  h->picture_structure & PICT_BOTTOM_FIELD,
440  .is_reference = h->nal_ref_idc != 0,
441  .complementary_field_pair = h->first_field && FIELD_PICTURE(h),
442  },
443  };
444 
445  hp->h264_pic_info = (VkVideoDecodeH264PictureInfoKHR) {
446  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PICTURE_INFO_KHR,
447  .pStdPictureInfo = &hp->h264pic,
448  };
449 
450  vp->decode_info = (VkVideoDecodeInfoKHR) {
451  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_INFO_KHR,
452  .pNext = &hp->h264_pic_info,
453  .flags = 0x0,
454  .pSetupReferenceSlot = &vp->ref_slot,
455  .referenceSlotCount = h->short_ref_count + h->long_ref_count,
456  .pReferenceSlots = vp->ref_slots,
457  .dstPictureResource = (VkVideoPictureResourceInfoKHR) {
458  .sType = VK_STRUCTURE_TYPE_VIDEO_PICTURE_RESOURCE_INFO_KHR,
459  .codedOffset = (VkOffset2D){ 0, 0 },
460  .codedExtent = (VkExtent2D){ pic->f->width, pic->f->height },
461  .baseArrayLayer = 0,
462  .imageViewBinding = vp->img_view_out,
463  },
464  };
465 
466  return 0;
467 }
468 
470  const uint8_t *data,
471  uint32_t size)
472 {
473  const H264Context *h = avctx->priv_data;
474  const H264SliceContext *sl = &h->slice_ctx[0];
475  H264VulkanDecodePicture *hp = h->cur_pic_ptr->hwaccel_picture_private;
476  FFVulkanDecodePicture *vp = &hp->vp;
477 
478  int err = ff_vk_decode_add_slice(avctx, vp, data, size, 1,
479  &hp->h264_pic_info.sliceCount,
480  &hp->h264_pic_info.pSliceOffsets);
481  if (err < 0)
482  return err;
483 
484  hp->h264pic.frame_num = sl->frame_num;
485  hp->h264pic.idr_pic_id = sl->idr_pic_id;
486 
487  /* Frame is only intra of all slices are marked as intra */
489  hp->h264pic.flags.is_intra = 0;
490 
491  return 0;
492 }
493 
495 {
496  const H264Context *h = avctx->priv_data;
497  H264Picture *pic = h->cur_pic_ptr;
500  FFVulkanDecodePicture *vp = &hp->vp;
502  AVFrame *rav[H264_MAX_PICTURE_COUNT] = { 0 };
503 
504  if (!hp->h264_pic_info.sliceCount)
505  return 0;
506 
507  if (!vp->slices_buf)
508  return AVERROR(EINVAL);
509 
510  if (!dec->session_params) {
511  int err = vk_h264_create_params(avctx, &dec->session_params);
512  if (err < 0)
513  return err;
514 
515  hp->h264pic.seq_parameter_set_id = pic->pps->sps_id;
516  hp->h264pic.pic_parameter_set_id = pic->pps->pps_id;
517  }
518 
519  for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
520  H264Picture *rp = hp->ref_src[i];
522 
523  rvp[i] = &rhp->vp;
524  rav[i] = hp->ref_src[i]->f;
525  }
526 
527  av_log(avctx, AV_LOG_VERBOSE, "Decoding frame, %"SIZE_SPECIFIER" bytes, %i slices\n",
528  vp->slices_size, hp->h264_pic_info.sliceCount);
529 
530  return ff_vk_decode_frame(avctx, pic->f, vp, rav, rvp);
531 }
532 
533 static void vk_h264_free_frame_priv(void *_hwctx, uint8_t *data)
534 {
535  AVHWDeviceContext *hwctx = _hwctx;
537 
538  /* Free frame resources, this also destroys the session parameters. */
539  ff_vk_decode_free_frame(hwctx, &hp->vp);
540 
541  /* Free frame context */
542  av_free(hp);
543 }
544 
546  .p.name = "h264_vulkan",
547  .p.type = AVMEDIA_TYPE_VIDEO,
548  .p.id = AV_CODEC_ID_H264,
549  .p.pix_fmt = AV_PIX_FMT_VULKAN,
550  .start_frame = &vk_h264_start_frame,
551  .decode_slice = &vk_h264_decode_slice,
552  .end_frame = &vk_h264_end_frame,
553  .free_frame_priv = &vk_h264_free_frame_priv,
554  .frame_priv_data_size = sizeof(H264VulkanDecodePicture),
557  .decode_params = &ff_vk_params_invalidate,
560  .frame_params = &ff_vk_frame_params,
561  .priv_data_size = sizeof(FFVulkanDecodeContext),
563 };
PICT_FRAME
#define PICT_FRAME
Definition: mpegutils.h:38
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:99
r
const char * r
Definition: vf_curves.c:126
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
H264Picture::f
AVFrame * f
Definition: h264dec.h:106
H264VulkanDecodePicture::h264_ref
StdVideoDecodeH264ReferenceInfo h264_ref
Definition: vulkan_h264.c:33
set_sps
static void set_sps(const SPS *sps, StdVideoH264ScalingLists *vksps_scaling, StdVideoH264HrdParameters *vksps_vui_header, StdVideoH264SequenceParameterSetVui *vksps_vui, StdVideoH264SequenceParameterSet *vksps)
Definition: vulkan_h264.c:136
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
H264VulkanDecodePicture::h264_pic_info
VkVideoDecodeH264PictureInfoKHR h264_pic_info
Definition: vulkan_h264.c:43
av_unused
#define av_unused
Definition: attributes.h:131
H264Picture::pps
const PPS * pps
Definition: h264dec.h:151
FFHWAccel::p
AVHWAccel p
The public AVHWAccel.
Definition: hwaccel_internal.h:37
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
AVFrame::width
int width
Definition: frame.h:412
level_idc
int level_idc
Definition: h264_levels.c:29
set_pps
static void set_pps(const PPS *pps, const SPS *sps, StdVideoH264ScalingLists *vkpps_scaling, StdVideoH264PictureParameterSet *vkpps)
Definition: vulkan_h264.c:244
data
const char data[16]
Definition: mxf.c:148
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
PICT_BOTTOM_FIELD
#define PICT_BOTTOM_FIELD
Definition: mpegutils.h:37
H264VulkanDecodePicture::vkh264_ref
VkVideoDecodeH264DpbSlotInfoKHR vkh264_ref
Definition: vulkan_h264.c:34
H264VulkanDecodePicture::h264pic
StdVideoDecodeH264PictureInfo h264pic
Definition: vulkan_h264.c:42
vk_h264_create_params
static int vk_h264_create_params(AVCodecContext *avctx, AVBufferRef **buf)
Definition: vulkan_h264.c:286
H264Picture::invalid_gap
int invalid_gap
Definition: h264dec.h:146
FFVulkanDecodeContext
Definition: vulkan_decode.h:56
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:376
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:173
FFVulkanDecodePicture::ref
VkVideoPictureResourceInfoKHR ref
Definition: vulkan_decode.h:87
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:58
H264Picture::frame_num
int frame_num
frame_num (raw frame_num from slice header)
Definition: h264dec.h:128
H264SliceContext
Definition: h264dec.h:171
ff_h264_vulkan_hwaccel
const FFHWAccel ff_h264_vulkan_hwaccel
Definition: vulkan_h264.c:545
FFHWAccel
Definition: hwaccel_internal.h:33
HWACCEL_CAP_THREAD_SAFE
#define HWACCEL_CAP_THREAD_SAFE
Definition: hwaccel_internal.h:31
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:329
FFVulkanDecodeShared
Definition: vulkan_decode.h:37
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
FFVulkanDecodePicture::img_view_out
VkImageView img_view_out
Definition: vulkan_decode.h:78
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
vk_h264_decode_slice
static int vk_h264_decode_slice(AVCodecContext *avctx, const uint8_t *data, uint32_t size)
Definition: vulkan_h264.c:469
FFVulkanDecodeContext::shared_ref
AVBufferRef * shared_ref
Definition: vulkan_decode.h:57
FFVulkanDecodePicture::refs
VkVideoPictureResourceInfoKHR refs[36]
Definition: vulkan_decode.h:91
H264VulkanDecodePicture
Definition: vulkan_h264.c:29
FFVulkanDecodePicture
Definition: vulkan_decode.h:74
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1066
PPS::pps_id
unsigned int pps_id
Definition: h264_ps.h:112
H264VulkanDecodePicture::h264_refs
StdVideoDecodeH264ReferenceInfo h264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:38
FIELD_PICTURE
#define FIELD_PICTURE(h)
Definition: h264dec.h:66
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts_bsf.c:365
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
PICT_TOP_FIELD
#define PICT_TOP_FIELD
Definition: mpegutils.h:36
H264VulkanDecodePicture::ref_src
H264Picture * ref_src[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:37
H264SliceContext::slice_type
int slice_type
Definition: h264dec.h:177
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
HWACCEL_CAP_ASYNC_SAFE
#define HWACCEL_CAP_ASYNC_SAFE
Header providing the internals of AVHWAccel.
Definition: hwaccel_internal.h:30
FFVulkanDecodePicture::img_view_ref
VkImageView img_view_ref
Definition: vulkan_decode.h:77
NULL
#define NULL
Definition: coverity.c:32
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:558
convert_to_vk_level_idc
static StdVideoH264LevelIdc convert_to_vk_level_idc(int level_idc)
Definition: vulkan_h264.c:110
SPS
Sequence parameter set.
Definition: h264_ps.h:45
AVCodecContext::internal
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:472
AV_PICTURE_TYPE_SI
@ AV_PICTURE_TYPE_SI
Switching Intra.
Definition: avutil.h:283
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:279
PPS
Picture parameter set.
Definition: h264_ps.h:111
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1095
MAX_PPS_COUNT
#define MAX_PPS_COUNT
Definition: h264_ps.h:39
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:1009
h264_ps.h
H264Picture::pic_id
int pic_id
pic_num (short -> no wrap version of pic_num, pic_num & max_pic_num; long -> long_pic_num)
Definition: h264dec.h:131
ff_zigzag_scan
const uint8_t ff_zigzag_scan[16+1]
Definition: mathtables.c:109
H264Picture::reference
int reference
Definition: h264dec.h:144
pps
static int FUNC() pps(CodedBitstreamContext *ctx, RWContext *rw, H264RawPPS *current)
Definition: cbs_h264_syntax_template.c:404
AVCodecInternal::hwaccel_priv_data
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:124
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
ff_vk_dec_h264_ext
const VkExtensionProperties ff_vk_dec_h264_ext
Definition: vulkan_h264.c:24
size
int size
Definition: twinvq_data.h:10344
h264_scaling_list8_order
const static int h264_scaling_list8_order[]
Definition: vulkan_h264.c:46
AVHWAccel::name
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:2129
h264dec.h
H264Context
H264Context.
Definition: h264dec.h:331
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:244
vk_h264_end_frame
static int vk_h264_end_frame(AVCodecContext *avctx)
Definition: vulkan_h264.c:494
H264SliceContext::frame_num
int frame_num
Definition: h264dec.h:319
H264VulkanDecodePicture::vp
FFVulkanDecodePicture vp
Definition: vulkan_h264.c:30
vk_h264_start_frame
static int vk_h264_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
Definition: vulkan_h264.c:354
FFVulkanDecodePicture::ref_slot
VkVideoReferenceSlotInfoKHR ref_slot
Definition: vulkan_decode.h:88
update_thread_context
the pkt_dts and pkt_pts fields in AVFrame will work as usual Restrictions on codec whose streams don t reset across will not work because their bitstreams cannot be decoded in parallel *The contents of buffers must not be read before as well as code calling up to before the decode process starts Call have update_thread_context() run it in the next thread. Add AV_CODEC_CAP_FRAME_THREADS to the codec capabilities. There will be very little speed gain at this point but it should work. If there are inter-frame dependencies
FFVulkanDecodePicture::ref_slots
VkVideoReferenceSlotInfoKHR ref_slots[36]
Definition: vulkan_decode.h:92
ff_zigzag_direct
const uint8_t ff_zigzag_direct[64]
Definition: mathtables.c:98
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:299
PPS::sps
const SPS * sps
Definition: h264_ps.h:143
vk_h264_fill_pict
static int vk_h264_fill_pict(AVCodecContext *avctx, H264Picture **ref_src, VkVideoReferenceSlotInfoKHR *ref_slot, VkVideoPictureResourceInfoKHR *ref, VkVideoDecodeH264DpbSlotInfoKHR *vkh264_ref, StdVideoDecodeH264ReferenceInfo *h264_ref, H264Picture *pic, int is_current, int is_field, int picture_structure, int dpb_slot_index)
Definition: vulkan_h264.c:48
sps
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
Definition: cbs_h264_syntax_template.c:260
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:237
SIZE_SPECIFIER
#define SIZE_SPECIFIER
Definition: internal.h:141
PPS::sps_id
unsigned int sps_id
Definition: h264_ps.h:113
AVCodecContext
main external API structure.
Definition: avcodec.h:437
AVFrame::height
int height
Definition: frame.h:412
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:61
H264Picture::field_poc
int field_poc[2]
top/bottom POC
Definition: h264dec.h:126
buffer
the frame and frame reference mechanism is intended to as much as expensive copies of that data while still allowing the filters to produce correct results The data is stored in buffers represented by AVFrame structures Several references can point to the same frame buffer
Definition: filter_design.txt:49
H264Picture
Definition: h264dec.h:105
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:110
MAX_SPS_COUNT
#define MAX_SPS_COUNT
Definition: h264_ps.h:38
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:67
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
vk_h264_free_frame_priv
static void vk_h264_free_frame_priv(void *_hwctx, uint8_t *data)
Definition: vulkan_h264.c:533
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:98
flush
void(* flush)(AVBSFContext *ctx)
Definition: dts2pts_bsf.c:367
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
H264Picture::hwaccel_picture_private
void * hwaccel_picture_private
hardware accelerator private data
Definition: h264dec.h:121
vulkan_decode.h
H264SliceContext::idr_pic_id
int idr_pic_id
Definition: h264dec.h:320
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
AVCodecContext::priv_data
void * priv_data
Definition: avcodec.h:464
FFVulkanDecodeContext::layered_dpb
int layered_dpb
Definition: vulkan_decode.h:62
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1110
uninit
static av_cold int uninit(AVCodecContext *avctx)
Definition: crystalhd.c:285
h
h
Definition: vp9dsp_template.c:2038
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:95
H264_MAX_PICTURE_COUNT
#define H264_MAX_PICTURE_COUNT
Definition: h264dec.h:48
H264VulkanDecodePicture::vkh264_refs
VkVideoDecodeH264DpbSlotInfoKHR vkh264_refs[H264_MAX_PICTURE_COUNT]
Definition: vulkan_h264.c:39
H264Picture::long_ref
int long_ref
1->long term reference 0->short term reference
Definition: h264dec.h:133