FFmpeg
vulkan_decode.c
Go to the documentation of this file.
1 /*
2  * This file is part of FFmpeg.
3  *
4  * FFmpeg is free software; you can redistribute it and/or
5  * modify it under the terms of the GNU Lesser General Public
6  * License as published by the Free Software Foundation; either
7  * version 2.1 of the License, or (at your option) any later version.
8  *
9  * FFmpeg is distributed in the hope that it will be useful,
10  * but WITHOUT ANY WARRANTY; without even the implied warranty of
11  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12  * Lesser General Public License for more details.
13  *
14  * You should have received a copy of the GNU Lesser General Public
15  * License along with FFmpeg; if not, write to the Free Software
16  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17  */
18 
19 #include "vulkan_video.h"
20 #include "vulkan_decode.h"
21 #include "config_components.h"
22 
23 #if CONFIG_H264_VULKAN_HWACCEL
24 extern const VkExtensionProperties ff_vk_dec_h264_ext;
25 #endif
26 #if CONFIG_HEVC_VULKAN_HWACCEL
27 extern const VkExtensionProperties ff_vk_dec_hevc_ext;
28 #endif
29 #if CONFIG_AV1_VULKAN_HWACCEL
30 extern const VkExtensionProperties ff_vk_dec_av1_ext;
31 #endif
32 
33 static const VkExtensionProperties *dec_ext[] = {
34 #if CONFIG_H264_VULKAN_HWACCEL
36 #endif
37 #if CONFIG_HEVC_VULKAN_HWACCEL
39 #endif
40 #if CONFIG_AV1_VULKAN_HWACCEL
42 #endif
43 };
44 
45 static const VkVideoProfileInfoKHR *get_video_profile(FFVulkanDecodeShared *ctx, enum AVCodecID codec_id)
46 {
47  const VkVideoProfileListInfoKHR *profile_list;
48 
49  VkStructureType profile_struct_type =
50  codec_id == AV_CODEC_ID_H264 ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR :
51  codec_id == AV_CODEC_ID_HEVC ? VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR :
53  0;
54 
55  profile_list = ff_vk_find_struct(ctx->s.hwfc->create_pnext,
56  VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
57  if (!profile_list)
58  return NULL;
59 
60  for (int i = 0; i < profile_list->profileCount; i++)
61  if (ff_vk_find_struct(profile_list->pProfiles[i].pNext, profile_struct_type))
62  return &profile_list->pProfiles[i];
63 
64  return NULL;
65 }
66 
68 {
69  int err;
70  FFVulkanDecodeContext *src_ctx = src->internal->hwaccel_priv_data;
72 
73  if (!dst_ctx->exec_pool.cmd_bufs) {
75 
76  const VkVideoProfileInfoKHR *profile = get_video_profile(ctx, dst->codec_id);
77  if (!profile) {
78  av_log(dst, AV_LOG_ERROR, "Video profile missing from frames context!");
79  return AVERROR(EINVAL);
80  }
81 
82  err = ff_vk_exec_pool_init(&ctx->s, &ctx->qf,
83  &dst_ctx->exec_pool,
84  src_ctx->exec_pool.pool_size,
85  src_ctx->exec_pool.nb_queries,
86  VK_QUERY_TYPE_RESULT_STATUS_ONLY_KHR, 0,
87  profile);
88  if (err < 0)
89  return err;
90  }
91 
92  err = av_buffer_replace(&dst_ctx->shared_ref, src_ctx->shared_ref);
93  if (err < 0)
94  return err;
95 
96  if (src_ctx->session_params) {
97  err = av_buffer_replace(&dst_ctx->session_params, src_ctx->session_params);
98  if (err < 0)
99  return err;
100  }
101 
102  dst_ctx->dedicated_dpb = src_ctx->dedicated_dpb;
103  dst_ctx->layered_dpb = src_ctx->layered_dpb;
104  dst_ctx->external_fg = src_ctx->external_fg;
105  dst_ctx->frame_id_alloc_mask = src_ctx->frame_id_alloc_mask;
106 
107  return 0;
108 }
109 
110 int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
111 {
114  return 0;
115 }
116 
117 static int vk_decode_create_view(FFVulkanDecodeShared *ctx, VkImageView *dst_view,
118  VkImageAspectFlags *aspect, AVVkFrame *src,
119  VkFormat vkf)
120 {
121  VkResult ret;
122  FFVulkanFunctions *vk = &ctx->s.vkfn;
123  VkImageAspectFlags aspect_mask = ff_vk_aspect_bits_from_vkfmt(vkf);
124 
125  VkSamplerYcbcrConversionInfo yuv_sampler_info = {
126  .sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_INFO,
127  .conversion = ctx->yuv_sampler,
128  };
129  VkImageViewCreateInfo img_view_create_info = {
130  .sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
131  .pNext = &yuv_sampler_info,
132  .viewType = VK_IMAGE_VIEW_TYPE_2D,
133  .format = vkf,
134  .image = src->img[0],
135  .components = (VkComponentMapping) {
136  .r = VK_COMPONENT_SWIZZLE_IDENTITY,
137  .g = VK_COMPONENT_SWIZZLE_IDENTITY,
138  .b = VK_COMPONENT_SWIZZLE_IDENTITY,
139  .a = VK_COMPONENT_SWIZZLE_IDENTITY,
140  },
141  .subresourceRange = (VkImageSubresourceRange) {
142  .aspectMask = VK_IMAGE_ASPECT_COLOR_BIT,
143  .baseArrayLayer = 0,
144  .layerCount = VK_REMAINING_ARRAY_LAYERS,
145  .levelCount = 1,
146  },
147  };
148 
149  ret = vk->CreateImageView(ctx->s.hwctx->act_dev, &img_view_create_info,
150  ctx->s.hwctx->alloc, dst_view);
151  if (ret != VK_SUCCESS)
152  return AVERROR_EXTERNAL;
153 
154  *aspect = aspect_mask;
155 
156  return 0;
157 }
158 
160 {
161  int err;
162  AVFrame *avf = av_frame_alloc();
163  if (!avf)
164  return NULL;
165 
166  err = av_hwframe_get_buffer(ctx->dpb_hwfc_ref, avf, 0x0);
167  if (err < 0)
168  av_frame_free(&avf);
169 
170  return avf;
171 }
172 
174  FFVulkanDecodePicture *vkpic, int is_current,
175  int alloc_dpb)
176 {
177  int err;
179  FFVulkanFunctions *vk = &ctx->s.vkfn;
180 
181  vkpic->slices_size = 0;
182 
183  /* If the decoder made a blank frame to make up for a missing ref, or the
184  * frame is the current frame so it's missing one, create a re-representation */
185  if (vkpic->img_view_ref)
186  return 0;
187 
188  vkpic->dpb_frame = NULL;
189  vkpic->img_view_ref = NULL;
190  vkpic->img_view_out = NULL;
191  vkpic->img_view_dest = NULL;
192 
193  vkpic->destroy_image_view = vk->DestroyImageView;
194  vkpic->wait_semaphores = vk->WaitSemaphores;
195 
196  if (dec->layered_dpb && alloc_dpb) {
197  vkpic->img_view_ref = ctx->layered_view;
198  vkpic->img_aspect_ref = ctx->layered_aspect;
199  } else if (alloc_dpb) {
200  AVHWFramesContext *dpb_frames = (AVHWFramesContext *)ctx->dpb_hwfc_ref->data;
201  AVVulkanFramesContext *dpb_hwfc = dpb_frames->hwctx;
202 
203  vkpic->dpb_frame = vk_get_dpb_pool(ctx);
204  if (!vkpic->dpb_frame)
205  return AVERROR(ENOMEM);
206 
207  err = vk_decode_create_view(ctx, &vkpic->img_view_ref,
208  &vkpic->img_aspect_ref,
209  (AVVkFrame *)vkpic->dpb_frame->data[0],
210  dpb_hwfc->format[0]);
211  if (err < 0)
212  return err;
213 
214  vkpic->img_view_dest = vkpic->img_view_ref;
215  }
216 
217  if (!alloc_dpb || is_current) {
219  AVVulkanFramesContext *hwfc = frames->hwctx;
220 
221  err = vk_decode_create_view(ctx, &vkpic->img_view_out,
222  &vkpic->img_aspect,
223  (AVVkFrame *)pic->data[0],
224  hwfc->format[0]);
225  if (err < 0)
226  return err;
227 
228  if (!alloc_dpb) {
229  vkpic->img_view_ref = vkpic->img_view_out;
230  vkpic->img_aspect_ref = vkpic->img_aspect;
231  }
232  }
233 
234  return 0;
235 }
236 
238  const uint8_t *data, size_t size, int add_startcode,
239  uint32_t *nb_slices, const uint32_t **offsets)
240 {
243 
244  static const uint8_t startcode_prefix[3] = { 0x0, 0x0, 0x1 };
245  const size_t startcode_len = add_startcode ? sizeof(startcode_prefix) : 0;
246  const int nb = *nb_slices;
247  uint8_t *slices;
248  uint32_t *slice_off;
249  FFVkVideoBuffer *vkbuf;
250 
251  size_t new_size = vp->slices_size + startcode_len + size +
252  ctx->caps.minBitstreamBufferSizeAlignment;
253  new_size = FFALIGN(new_size, ctx->caps.minBitstreamBufferSizeAlignment);
254 
255  slice_off = av_fast_realloc(dec->slice_off, &dec->slice_off_max,
256  (nb + 1)*sizeof(slice_off));
257  if (!slice_off)
258  return AVERROR(ENOMEM);
259 
260  *offsets = dec->slice_off = slice_off;
261  slice_off[nb] = vp->slices_size;
262 
263  vkbuf = vp->slices_buf ? (FFVkVideoBuffer *)vp->slices_buf->data : NULL;
264  if (!vkbuf || vkbuf->buf.size < new_size) {
265  int err;
266  AVBufferRef *new_ref;
267  FFVkVideoBuffer *new_buf;
268  err = ff_vk_video_get_buffer(&ctx->s, &ctx->common, &new_ref,
269  VK_BUFFER_USAGE_VIDEO_DECODE_SRC_BIT_KHR,
270  ctx->s.hwfc->create_pnext, new_size);
271  if (err < 0)
272  return err;
273 
274  new_buf = (FFVkVideoBuffer *)new_ref->data;
275 
276  /* Copy data from the old buffer */
277  if (vkbuf) {
278  memcpy(new_buf->mem, vkbuf->mem, vp->slices_size);
279  av_buffer_unref(&vp->slices_buf);
280  }
281 
282  vp->slices_buf = new_ref;
283  vkbuf = new_buf;
284  }
285  slices = vkbuf->mem;
286 
287  /* Startcode */
288  memcpy(slices + vp->slices_size, startcode_prefix, startcode_len);
289 
290  /* Slice data */
291  memcpy(slices + vp->slices_size + startcode_len, data, size);
292 
293  *nb_slices = nb + 1;
294  vp->slices_size += startcode_len + size;
295 
296  return 0;
297 }
298 
300 {
303 
304  FFVulkanFunctions *vk = &ctx->s.vkfn;
305  VkVideoBeginCodingInfoKHR decode_start = {
306  .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
307  .videoSession = ctx->common.session,
308  .videoSessionParameters = ctx->empty_session_params,
309  };
310  VkVideoCodingControlInfoKHR decode_ctrl = {
311  .sType = VK_STRUCTURE_TYPE_VIDEO_CODING_CONTROL_INFO_KHR,
312  .flags = VK_VIDEO_CODING_CONTROL_RESET_BIT_KHR,
313  };
314  VkVideoEndCodingInfoKHR decode_end = {
315  .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
316  };
317 
318  VkCommandBuffer cmd_buf;
319  FFVkExecContext *exec = ff_vk_exec_get(&dec->exec_pool);
320  ff_vk_exec_start(&ctx->s, exec);
321  cmd_buf = exec->buf;
322 
323  vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start);
324  vk->CmdControlVideoCodingKHR(cmd_buf, &decode_ctrl);
325  vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end);
326  ff_vk_exec_submit(&ctx->s, exec);
327 }
328 
330  AVFrame *pic, FFVulkanDecodePicture *vp,
331  AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
332 {
333  int err;
334  VkResult ret;
335  VkCommandBuffer cmd_buf;
336  FFVkVideoBuffer *sd_buf;
337 
340  FFVulkanFunctions *vk = &ctx->s.vkfn;
341 
342  /* Output */
343  AVVkFrame *vkf = (AVVkFrame *)pic->buf[0]->data;
344 
345  /* Quirks */
346  const int layered_dpb = dec->layered_dpb;
347 
348  VkVideoSessionParametersKHR *par = (VkVideoSessionParametersKHR *)dec->session_params->data;
349  VkVideoBeginCodingInfoKHR decode_start = {
350  .sType = VK_STRUCTURE_TYPE_VIDEO_BEGIN_CODING_INFO_KHR,
351  .videoSession = ctx->common.session,
352  .videoSessionParameters = *par,
353  .referenceSlotCount = vp->decode_info.referenceSlotCount,
354  .pReferenceSlots = vp->decode_info.pReferenceSlots,
355  };
356  VkVideoEndCodingInfoKHR decode_end = {
357  .sType = VK_STRUCTURE_TYPE_VIDEO_END_CODING_INFO_KHR,
358  };
359 
360  VkImageMemoryBarrier2 img_bar[37];
361  int nb_img_bar = 0;
362  size_t data_size = FFALIGN(vp->slices_size,
363  ctx->caps.minBitstreamBufferSizeAlignment);
364 
365  FFVkExecContext *exec = ff_vk_exec_get(&dec->exec_pool);
366 
367  /* The current decoding reference has to be bound as an inactive reference */
368  VkVideoReferenceSlotInfoKHR *cur_vk_ref;
369  cur_vk_ref = (void *)&decode_start.pReferenceSlots[decode_start.referenceSlotCount];
370  cur_vk_ref[0] = vp->ref_slot;
371  cur_vk_ref[0].slotIndex = -1;
372  decode_start.referenceSlotCount++;
373 
374  if (dec->exec_pool.nb_queries) {
375  int64_t prev_sub_res = 0;
376  ff_vk_exec_wait(&ctx->s, exec);
377  ret = ff_vk_exec_get_query(&ctx->s, exec, NULL, &prev_sub_res);
378  if (ret != VK_NOT_READY && ret != VK_SUCCESS) {
379  av_log(avctx, AV_LOG_ERROR, "Unable to perform query: %s!\n",
380  ff_vk_ret2str(ret));
381  return AVERROR_EXTERNAL;
382  }
383 
384  if (ret == VK_SUCCESS)
385  av_log(avctx, prev_sub_res < 0 ? AV_LOG_ERROR : AV_LOG_DEBUG,
386  "Result of previous frame decoding: %"PRId64"\n", prev_sub_res);
387  }
388 
389  sd_buf = (FFVkVideoBuffer *)vp->slices_buf->data;
390 
391  /* Flush if needed */
392  if (!(sd_buf->buf.flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT)) {
393  VkMappedMemoryRange flush_buf = {
394  .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
395  .memory = sd_buf->buf.mem,
396  .offset = 0,
397  .size = FFALIGN(vp->slices_size,
398  ctx->s.props.properties.limits.nonCoherentAtomSize),
399  };
400 
401  ret = vk->FlushMappedMemoryRanges(ctx->s.hwctx->act_dev, 1, &flush_buf);
402  if (ret != VK_SUCCESS) {
403  av_log(avctx, AV_LOG_ERROR, "Failed to flush memory: %s\n",
404  ff_vk_ret2str(ret));
405  return AVERROR_EXTERNAL;
406  }
407  }
408 
409  vp->decode_info.srcBuffer = sd_buf->buf.buf;
410  vp->decode_info.srcBufferOffset = 0;
411  vp->decode_info.srcBufferRange = data_size;
412 
413  /* Start command buffer recording */
414  err = ff_vk_exec_start(&ctx->s, exec);
415  if (err < 0)
416  return err;
417  cmd_buf = exec->buf;
418 
419  /* Slices */
420  err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &vp->slices_buf, 1, 0);
421  if (err < 0)
422  return err;
423  vp->slices_buf = NULL; /* Owned by the exec buffer from now on */
424 
425  /* Parameters */
426  err = ff_vk_exec_add_dep_buf(&ctx->s, exec, &dec->session_params, 1, 1);
427  if (err < 0)
428  return err;
429 
430  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, pic,
431  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
432  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
433  if (err < 0)
434  return err;
435 
436  err = ff_vk_exec_mirror_sem_value(&ctx->s, exec, &vp->sem, &vp->sem_value,
437  pic);
438  if (err < 0)
439  return err;
440 
441  /* Output image - change layout, as it comes from a pool */
442  img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
443  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
444  .pNext = NULL,
445  .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
446  .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
447  .srcAccessMask = VK_ACCESS_2_NONE,
448  .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
449  .oldLayout = vkf->layout[0],
450  .newLayout = vp->dpb_frame ? VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR :
451  VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR, /* Spec, 07252 utter madness */
452  .srcQueueFamilyIndex = vkf->queue_family[0],
453  .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
454  .image = vkf->img[0],
455  .subresourceRange = (VkImageSubresourceRange) {
456  .aspectMask = vp->img_aspect,
457  .layerCount = 1,
458  .levelCount = 1,
459  },
460  };
461  ff_vk_exec_update_frame(&ctx->s, exec, pic,
462  &img_bar[nb_img_bar], &nb_img_bar);
463 
464  /* Reference for the current image, if existing and not layered */
465  if (vp->dpb_frame) {
466  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, vp->dpb_frame,
467  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
468  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
469  if (err < 0)
470  return err;
471  }
472 
473  if (!layered_dpb) {
474  /* All references (apart from the current) for non-layered refs */
475 
476  for (int i = 0; i < vp->decode_info.referenceSlotCount; i++) {
477  AVFrame *ref_frame = rpic[i];
478  FFVulkanDecodePicture *rvp = rvkp[i];
479  AVFrame *ref = rvp->dpb_frame ? rvp->dpb_frame : ref_frame;
480 
481  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ref,
482  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
483  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
484  if (err < 0)
485  return err;
486 
487  if (err == 0) {
488  err = ff_vk_exec_mirror_sem_value(&ctx->s, exec,
489  &rvp->sem, &rvp->sem_value,
490  ref);
491  if (err < 0)
492  return err;
493  }
494 
495  if (!rvp->dpb_frame) {
496  AVVkFrame *rvkf = (AVVkFrame *)ref->data[0];
497 
498  img_bar[nb_img_bar] = (VkImageMemoryBarrier2) {
499  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2,
500  .pNext = NULL,
501  .srcStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
502  .dstStageMask = VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
503  .srcAccessMask = VK_ACCESS_2_NONE,
504  .dstAccessMask = VK_ACCESS_2_VIDEO_DECODE_READ_BIT_KHR |
505  VK_ACCESS_2_VIDEO_DECODE_WRITE_BIT_KHR,
506  .oldLayout = rvkf->layout[0],
507  .newLayout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR,
508  .srcQueueFamilyIndex = rvkf->queue_family[0],
509  .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
510  .image = rvkf->img[0],
511  .subresourceRange = (VkImageSubresourceRange) {
512  .aspectMask = rvp->img_aspect_ref,
513  .layerCount = 1,
514  .levelCount = 1,
515  },
516  };
517  ff_vk_exec_update_frame(&ctx->s, exec, ref,
518  &img_bar[nb_img_bar], &nb_img_bar);
519  }
520  }
521  } else if (vp->decode_info.referenceSlotCount ||
522  vp->img_view_out != vp->img_view_ref) {
523  /* Single barrier for a single layered ref */
524  err = ff_vk_exec_add_dep_frame(&ctx->s, exec, ctx->layered_frame,
525  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR,
526  VK_PIPELINE_STAGE_2_VIDEO_DECODE_BIT_KHR);
527  if (err < 0)
528  return err;
529  }
530 
531  /* Change image layout */
532  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
533  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
534  .dependencyFlags = VK_DEPENDENCY_BY_REGION_BIT,
535  .pImageMemoryBarriers = img_bar,
536  .imageMemoryBarrierCount = nb_img_bar,
537  });
538 
539  /* Start, use parameters, decode and end decoding */
540  vk->CmdBeginVideoCodingKHR(cmd_buf, &decode_start);
541 
542  /* Start status query */
543  if (dec->exec_pool.nb_queries)
544  vk->CmdBeginQuery(cmd_buf, dec->exec_pool.query_pool, exec->query_idx + 0, 0);
545 
546  vk->CmdDecodeVideoKHR(cmd_buf, &vp->decode_info);
547 
548  /* End status query */
549  if (dec->exec_pool.nb_queries)
550  vk->CmdEndQuery(cmd_buf, dec->exec_pool.query_pool, exec->query_idx + 0);
551 
552  vk->CmdEndVideoCodingKHR(cmd_buf, &decode_end);
553 
554  /* End recording and submit for execution */
555  return ff_vk_exec_submit(&ctx->s, exec);
556 }
557 
559 {
560  AVVulkanDeviceContext *hwctx = dev_ctx->hwctx;
561 
562  VkSemaphoreWaitInfo sem_wait = (VkSemaphoreWaitInfo) {
563  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
564  .pSemaphores = &vp->sem,
565  .pValues = &vp->sem_value,
566  .semaphoreCount = 1,
567  };
568 
569  /* We do not have to lock the frame here because we're not interested
570  * in the actual current semaphore value, but only that it's later than
571  * the time we submitted the image for decoding. */
572  if (vp->sem)
573  vp->wait_semaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
574 
575  /* Free slices data */
577 
578  /* Destroy image view (out) */
579  if (vp->img_view_out && vp->img_view_out != vp->img_view_dest)
580  vp->destroy_image_view(hwctx->act_dev, vp->img_view_out, hwctx->alloc);
581 
582  /* Destroy image view (ref, unlayered) */
583  if (vp->img_view_dest)
584  vp->destroy_image_view(hwctx->act_dev, vp->img_view_dest, hwctx->alloc);
585 
586  av_frame_free(&vp->dpb_frame);
587 }
588 
589 static void free_common(void *opaque, uint8_t *data)
590 {
592  FFVulkanContext *s = &ctx->s;
593  FFVulkanFunctions *vk = &ctx->s.vkfn;
594 
595  /* Destroy layered view */
596  if (ctx->layered_view)
597  vk->DestroyImageView(s->hwctx->act_dev, ctx->layered_view, s->hwctx->alloc);
598 
599  /* This also frees all references from this pool */
600  av_frame_free(&ctx->layered_frame);
601  av_buffer_unref(&ctx->dpb_hwfc_ref);
602 
603  /* Destroy parameters */
604  if (ctx->empty_session_params)
605  vk->DestroyVideoSessionParametersKHR(s->hwctx->act_dev,
606  ctx->empty_session_params,
607  s->hwctx->alloc);
608 
609  ff_vk_video_common_uninit(s, &ctx->common);
610 
611  if (ctx->yuv_sampler)
612  vk->DestroySamplerYcbcrConversion(s->hwctx->act_dev, ctx->yuv_sampler,
613  s->hwctx->alloc);
614 
615  ff_vk_uninit(s);
616 
617  av_free(ctx);
618 }
619 
620 static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
621 {
622  int err;
625  AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
626  AVVulkanDeviceContext *hwctx = device->hwctx;
628 
629  if (dec->shared_ref)
630  return 0;
631 
632  ctx = av_mallocz(sizeof(*ctx));
633  if (!ctx)
634  return AVERROR(ENOMEM);
635 
636  dec->shared_ref = av_buffer_create((uint8_t *)ctx, sizeof(*ctx),
637  free_common, NULL, 0);
638  if (!dec->shared_ref) {
639  av_free(ctx);
640  return AVERROR(ENOMEM);
641  }
642 
644 
645  ctx->s.extensions = ff_vk_extensions_to_mask(hwctx->enabled_dev_extensions,
647 
648  if (!(ctx->s.extensions & FF_VK_EXT_VIDEO_DECODE_QUEUE)) {
649  av_log(avctx, AV_LOG_ERROR, "Device does not support the %s extension!\n",
650  VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME);
652  return AVERROR(ENOSYS);
653  }
654 
655  err = ff_vk_load_functions(device, &ctx->s.vkfn, ctx->s.extensions, 1, 1);
656  if (err < 0) {
658  return err;
659  }
660 
661  return 0;
662 }
663 
664 static VkResult vulkan_setup_profile(AVCodecContext *avctx,
666  AVVulkanDeviceContext *hwctx,
667  FFVulkanFunctions *vk,
668  const struct FFVkCodecMap *vk_codec,
669  VkVideoDecodeH264CapabilitiesKHR *h264_caps,
670  VkVideoDecodeH265CapabilitiesKHR *h265_caps,
672  VkVideoCapabilitiesKHR *caps,
673  VkVideoDecodeCapabilitiesKHR *dec_caps,
674  int cur_profile)
675 {
676  VkVideoDecodeUsageInfoKHR *usage = &prof->usage;
677  VkVideoProfileInfoKHR *profile = &prof->profile;
678  VkVideoProfileListInfoKHR *profile_list = &prof->profile_list;
679 
680  VkVideoDecodeH264ProfileInfoKHR *h264_profile = &prof->h264_profile;
681  VkVideoDecodeH265ProfileInfoKHR *h265_profile = &prof->h265_profile;
682  VkVideoDecodeAV1ProfileInfoMESA *av1_profile = &prof->av1_profile;
683 
685  if (!desc)
686  return AVERROR(EINVAL);
687 
688  if (avctx->codec_id == AV_CODEC_ID_H264) {
689  dec_caps->pNext = h264_caps;
690  usage->pNext = h264_profile;
691  h264_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_PROFILE_INFO_KHR;
692 
693  /* Vulkan transmits all the constrant_set flags, rather than wanting them
694  * merged in the profile IDC */
695  h264_profile->stdProfileIdc = cur_profile & ~(AV_PROFILE_H264_CONSTRAINED |
697 
698  h264_profile->pictureLayout = avctx->field_order == AV_FIELD_UNKNOWN ||
700  VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_PROGRESSIVE_KHR :
701  VK_VIDEO_DECODE_H264_PICTURE_LAYOUT_INTERLACED_INTERLEAVED_LINES_BIT_KHR;
702  } else if (avctx->codec_id == AV_CODEC_ID_H265) {
703  dec_caps->pNext = h265_caps;
704  usage->pNext = h265_profile;
705  h265_profile->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_PROFILE_INFO_KHR;
706  h265_profile->stdProfileIdc = cur_profile;
707  } else if (avctx->codec_id == AV_CODEC_ID_AV1) {
708  dec_caps->pNext = av1_caps;
709  usage->pNext = av1_profile;
711  av1_profile->stdProfileIdc = cur_profile;
712  }
713 
714  usage->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_USAGE_INFO_KHR;
715  usage->videoUsageHints = VK_VIDEO_DECODE_USAGE_DEFAULT_KHR;
716 
717  profile->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_INFO_KHR;
718  profile->pNext = usage;
719  profile->videoCodecOperation = vk_codec->decode_op;
720  profile->chromaSubsampling = ff_vk_subsampling_from_av_desc(desc);
721  profile->lumaBitDepth = ff_vk_depth_from_av_depth(desc->comp[0].depth);
722  profile->chromaBitDepth = profile->lumaBitDepth;
723 
724  profile_list->sType = VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR;
725  profile_list->profileCount = 1;
726  profile_list->pProfiles = profile;
727 
728  /* Get the capabilities of the decoder for the given profile */
729  caps->sType = VK_STRUCTURE_TYPE_VIDEO_CAPABILITIES_KHR;
730  caps->pNext = dec_caps;
731  dec_caps->sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_CAPABILITIES_KHR;
732  /* dec_caps->pNext already filled in */
733 
734  return vk->GetPhysicalDeviceVideoCapabilitiesKHR(hwctx->phys_dev, profile,
735  caps);
736 }
737 
738 static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref,
739  enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt,
741  int *dpb_dedicate)
742 {
743  VkResult ret;
744  int max_level, base_profile, cur_profile;
745  const struct FFVkCodecMap *vk_codec = &ff_vk_codec_map[avctx->codec_id];
747  AVHWDeviceContext *device = (AVHWDeviceContext *)frames->device_ref->data;
748  AVVulkanDeviceContext *hwctx = device->hwctx;
749  enum AVPixelFormat source_format;
750  enum AVPixelFormat best_format;
751  VkFormat best_vkfmt;
752 
755  FFVulkanFunctions *vk = &ctx->s.vkfn;
756 
757  VkVideoCapabilitiesKHR *caps = &ctx->caps;
758  VkVideoDecodeCapabilitiesKHR *dec_caps = &ctx->dec_caps;
759 
760  VkVideoDecodeH264CapabilitiesKHR h264_caps = {
761  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_CAPABILITIES_KHR,
762  };
763  VkVideoDecodeH265CapabilitiesKHR h265_caps = {
764  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_CAPABILITIES_KHR,
765  };
768  };
769 
770  VkPhysicalDeviceVideoFormatInfoKHR fmt_info = {
771  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_FORMAT_INFO_KHR,
772  .pNext = &prof->profile_list,
773  };
774  VkVideoFormatPropertiesKHR *ret_info;
775  uint32_t nb_out_fmts = 0;
776 
777  if (!vk_codec->decode_op || !vk_codec->decode_extension) {
778  av_log(avctx, AV_LOG_ERROR, "Unsupported codec for Vulkan decoding: %s!\n",
779  avcodec_get_name(avctx->codec_id));
780  return AVERROR(ENOSYS);
781  } else if (!(vk_codec->decode_extension & ctx->s.extensions)) {
782  av_log(avctx, AV_LOG_ERROR, "Device does not support decoding %s!\n",
783  avcodec_get_name(avctx->codec_id));
784  return AVERROR(ENOSYS);
785  }
786 
787  cur_profile = avctx->profile;
788  base_profile = avctx->codec_id == AV_CODEC_ID_H264 ? AV_PROFILE_H264_CONSTRAINED_BASELINE :
789  avctx->codec_id == AV_CODEC_ID_H265 ? AV_PROFILE_HEVC_MAIN :
790  avctx->codec_id == AV_CODEC_ID_AV1 ? STD_VIDEO_AV1_MESA_PROFILE_MAIN :
791  0;
792 
793  ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_codec,
794  &h264_caps,
795  &h265_caps,
796  &av1_caps,
797  caps,
798  dec_caps,
799  cur_profile);
800  if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR &&
802  avctx->profile != base_profile) {
803  av_log(avctx, AV_LOG_VERBOSE, "%s profile %s not supported, attempting "
804  "again with profile %s\n",
805  avcodec_get_name(avctx->codec_id),
806  avcodec_profile_name(avctx->codec_id, cur_profile),
807  avcodec_profile_name(avctx->codec_id, base_profile));
808  cur_profile = base_profile;
809  ret = vulkan_setup_profile(avctx, prof, hwctx, vk, vk_codec,
810  &h264_caps,
811  &h265_caps,
812  &av1_caps,
813  caps,
814  dec_caps,
815  cur_profile);
816  }
817 
818  if (ret == VK_ERROR_VIDEO_PROFILE_OPERATION_NOT_SUPPORTED_KHR) {
819  av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
820  "%s profile \"%s\" not supported!\n",
821  avcodec_get_name(avctx->codec_id),
822  avcodec_profile_name(avctx->codec_id, cur_profile));
823  return AVERROR(EINVAL);
824  } else if (ret == VK_ERROR_VIDEO_PROFILE_FORMAT_NOT_SUPPORTED_KHR) {
825  av_log(avctx, AV_LOG_VERBOSE, "Unable to initialize video session: "
826  "format (%s) not supported!\n",
827  av_get_pix_fmt_name(avctx->sw_pix_fmt));
828  return AVERROR(EINVAL);
829  } else if (ret == VK_ERROR_FEATURE_NOT_PRESENT ||
830  ret == VK_ERROR_FORMAT_NOT_SUPPORTED) {
831  return AVERROR(EINVAL);
832  } else if (ret != VK_SUCCESS) {
833  return AVERROR_EXTERNAL;
834  }
835 
836  max_level = avctx->codec_id == AV_CODEC_ID_H264 ? ff_vk_h264_level_to_av(h264_caps.maxLevelIdc) :
837  avctx->codec_id == AV_CODEC_ID_H265 ? ff_vk_h265_level_to_av(h265_caps.maxLevelIdc) :
838  avctx->codec_id == AV_CODEC_ID_AV1 ? av1_caps.maxLevelIdc :
839  0;
840 
841  av_log(avctx, AV_LOG_VERBOSE, "Decoder capabilities for %s profile \"%s\":\n",
842  avcodec_get_name(avctx->codec_id),
843  avcodec_profile_name(avctx->codec_id, cur_profile));
844  av_log(avctx, AV_LOG_VERBOSE, " Maximum level: %i (stream %i)\n",
845  max_level, avctx->level);
846  av_log(avctx, AV_LOG_VERBOSE, " Width: from %i to %i\n",
847  caps->minCodedExtent.width, caps->maxCodedExtent.width);
848  av_log(avctx, AV_LOG_VERBOSE, " Height: from %i to %i\n",
849  caps->minCodedExtent.height, caps->maxCodedExtent.height);
850  av_log(avctx, AV_LOG_VERBOSE, " Width alignment: %i\n",
851  caps->pictureAccessGranularity.width);
852  av_log(avctx, AV_LOG_VERBOSE, " Height alignment: %i\n",
853  caps->pictureAccessGranularity.height);
854  av_log(avctx, AV_LOG_VERBOSE, " Bitstream offset alignment: %"PRIu64"\n",
855  caps->minBitstreamBufferOffsetAlignment);
856  av_log(avctx, AV_LOG_VERBOSE, " Bitstream size alignment: %"PRIu64"\n",
857  caps->minBitstreamBufferSizeAlignment);
858  av_log(avctx, AV_LOG_VERBOSE, " Maximum references: %u\n",
859  caps->maxDpbSlots);
860  av_log(avctx, AV_LOG_VERBOSE, " Maximum active references: %u\n",
861  caps->maxActiveReferencePictures);
862  av_log(avctx, AV_LOG_VERBOSE, " Codec header name: '%s' (driver), '%s' (compiled)\n",
863  caps->stdHeaderVersion.extensionName,
864  dec_ext[avctx->codec_id]->extensionName);
865  av_log(avctx, AV_LOG_VERBOSE, " Codec header version: %i.%i.%i (driver), %i.%i.%i (compiled)\n",
866  CODEC_VER(caps->stdHeaderVersion.specVersion),
867  CODEC_VER(dec_ext[avctx->codec_id]->specVersion));
868  av_log(avctx, AV_LOG_VERBOSE, " Decode modes:%s%s%s\n",
869  dec_caps->flags ? "" :
870  " invalid",
871  dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR ?
872  " reuse_dst_dpb" : "",
873  dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR ?
874  " dedicated_dpb" : "");
875  av_log(avctx, AV_LOG_VERBOSE, " Capability flags:%s%s%s\n",
876  caps->flags ? "" :
877  " none",
878  caps->flags & VK_VIDEO_CAPABILITY_PROTECTED_CONTENT_BIT_KHR ?
879  " protected" : "",
880  caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR ?
881  " separate_references" : "");
882 
883  /* Check if decoding is possible with the given parameters */
884  if (avctx->width < caps->minCodedExtent.width ||
885  avctx->height < caps->minCodedExtent.height ||
886  avctx->width > caps->maxCodedExtent.width ||
887  avctx->height > caps->maxCodedExtent.height)
888  return AVERROR(EINVAL);
889 
890  if (!(avctx->hwaccel_flags & AV_HWACCEL_FLAG_IGNORE_LEVEL) &&
891  avctx->level > max_level)
892  return AVERROR(EINVAL);
893 
894  /* Some basic sanity checking */
895  if (!(dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
896  VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR))) {
897  av_log(avctx, AV_LOG_ERROR, "Buggy driver signals invalid decoding mode: neither "
898  "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR nor "
899  "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR are set!\n");
900  return AVERROR_EXTERNAL;
901  } else if ((dec_caps->flags & (VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR |
902  VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR) ==
903  VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR) &&
904  !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR)) {
905  av_log(avctx, AV_LOG_ERROR, "Cannot initialize Vulkan decoding session, buggy driver: "
906  "VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR set "
907  "but VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR is unset!\n");
908  return AVERROR_EXTERNAL;
909  } else if (!(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR) &&
910  avctx->codec_id == AV_CODEC_ID_AV1) {
911  av_log(avctx, AV_LOG_ERROR, "Cannot initialize Vulkan decoding session, buggy driver: "
912  "codec is AV1, but VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_DISTINCT_BIT_KHR isn't set!\n");
913  return AVERROR_EXTERNAL;
914  }
915 
916  /* TODO: make dedicated_dpb tunable */
917  dec->dedicated_dpb = !(dec_caps->flags & VK_VIDEO_DECODE_CAPABILITY_DPB_AND_OUTPUT_COINCIDE_BIT_KHR);
918  dec->layered_dpb = !(caps->flags & VK_VIDEO_CAPABILITY_SEPARATE_REFERENCE_IMAGES_BIT_KHR);
920 
921  if (dec->dedicated_dpb) {
922  fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
923  } else {
924  fmt_info.imageUsage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
925  VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR |
926  VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
927  VK_IMAGE_USAGE_SAMPLED_BIT;
928  }
929 
930  /* Get the format of the images necessary */
931  ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
932  &fmt_info,
933  &nb_out_fmts, NULL);
934  if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
935  (!nb_out_fmts && ret == VK_SUCCESS)) {
936  return AVERROR(EINVAL);
937  } else if (ret != VK_SUCCESS) {
938  av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
939  ff_vk_ret2str(ret));
940  return AVERROR_EXTERNAL;
941  }
942 
943  ret_info = av_mallocz(sizeof(*ret_info)*nb_out_fmts);
944  if (!ret_info)
945  return AVERROR(ENOMEM);
946 
947  for (int i = 0; i < nb_out_fmts; i++)
948  ret_info[i].sType = VK_STRUCTURE_TYPE_VIDEO_FORMAT_PROPERTIES_KHR;
949 
950  ret = vk->GetPhysicalDeviceVideoFormatPropertiesKHR(hwctx->phys_dev,
951  &fmt_info,
952  &nb_out_fmts, ret_info);
953  if (ret == VK_ERROR_FORMAT_NOT_SUPPORTED ||
954  (!nb_out_fmts && ret == VK_SUCCESS)) {
955  av_free(ret_info);
956  return AVERROR(EINVAL);
957  } else if (ret != VK_SUCCESS) {
958  av_log(avctx, AV_LOG_ERROR, "Unable to get Vulkan format properties: %s!\n",
959  ff_vk_ret2str(ret));
960  av_free(ret_info);
961  return AVERROR_EXTERNAL;
962  }
963 
964  /* Find a format to use */
965  *pix_fmt = best_format = AV_PIX_FMT_NONE;
966  *vk_fmt = best_vkfmt = VK_FORMAT_UNDEFINED;
967  source_format = avctx->sw_pix_fmt;
968 
969  av_log(avctx, AV_LOG_DEBUG, "Choosing best pixel format for decoding from %i:\n", nb_out_fmts);
970  for (int i = 0; i < nb_out_fmts; i++) {
972  if (tmp == AV_PIX_FMT_NONE) {
973  av_log(avctx, AV_LOG_WARNING, "Invalid/unknown Vulkan format %i!\n", ret_info[i].format);
974  continue;
975  }
976 
977  best_format = av_find_best_pix_fmt_of_2(tmp, best_format, source_format, 0, NULL);
978  if (tmp == best_format)
979  best_vkfmt = ret_info[i].format;
980 
981  av_log(avctx, AV_LOG_DEBUG, " %s%s (Vulkan ID: %i)\n",
982  av_get_pix_fmt_name(tmp), tmp == best_format ? "*" : "",
983  ret_info[i].format);
984  }
985 
986  av_free(ret_info);
987 
988  if (best_format == AV_PIX_FMT_NONE) {
989  av_log(avctx, AV_LOG_ERROR, "No valid/compatible pixel format found for decoding!\n");
990  return AVERROR(EINVAL);
991  } else {
992  av_log(avctx, AV_LOG_VERBOSE, "Chosen frame pixfmt: %s (Vulkan ID: %i)\n",
993  av_get_pix_fmt_name(best_format), best_vkfmt);
994  }
995 
996  *pix_fmt = best_format;
997  *vk_fmt = best_vkfmt;
998 
999  *dpb_dedicate = dec->dedicated_dpb;
1000 
1001  return 0;
1002 }
1003 
1005 {
1006  av_free(hwfc->user_opaque);
1007 }
1008 
1009 int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
1010 {
1011  VkFormat vkfmt;
1012  int err, dedicated_dpb;
1013  AVHWFramesContext *frames_ctx = (AVHWFramesContext*)hw_frames_ctx->data;
1014  AVVulkanFramesContext *hwfc = frames_ctx->hwctx;
1017 
1018  frames_ctx->sw_format = AV_PIX_FMT_NONE;
1019 
1020  err = vulkan_decode_bootstrap(avctx, hw_frames_ctx);
1021  if (err < 0)
1022  return err;
1023 
1024  prof = av_mallocz(sizeof(FFVulkanDecodeProfileData));
1025  if (!prof)
1026  return AVERROR(ENOMEM);
1027 
1028  err = vulkan_decode_get_profile(avctx, hw_frames_ctx,
1029  &frames_ctx->sw_format, &vkfmt,
1030  prof, &dedicated_dpb);
1031  if (err < 0) {
1032  av_free(prof);
1033  return err;
1034  }
1035 
1036  frames_ctx->user_opaque = prof;
1037  frames_ctx->free = free_profile_data;
1038 
1039  frames_ctx->width = avctx->width;
1040  frames_ctx->height = avctx->height;
1041  frames_ctx->format = AV_PIX_FMT_VULKAN;
1042 
1043  hwfc->format[0] = vkfmt;
1044  hwfc->create_pnext = &prof->profile_list;
1045  hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
1046  hwfc->usage = VK_IMAGE_USAGE_TRANSFER_SRC_BIT |
1047  VK_IMAGE_USAGE_SAMPLED_BIT |
1048  VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR;
1049 
1050  if (!dec->dedicated_dpb)
1051  hwfc->usage |= VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR;
1052 
1053  return err;
1054 }
1055 
1056 static void vk_decode_free_params(void *opaque, uint8_t *data)
1057 {
1058  FFVulkanDecodeShared *ctx = opaque;
1059  FFVulkanFunctions *vk = &ctx->s.vkfn;
1060  VkVideoSessionParametersKHR *par = (VkVideoSessionParametersKHR *)data;
1061  vk->DestroyVideoSessionParametersKHR(ctx->s.hwctx->act_dev, *par,
1062  ctx->s.hwctx->alloc);
1063  av_free(par);
1064 }
1065 
1067  const VkVideoSessionParametersCreateInfoKHR *session_params_create)
1068 {
1069  VkVideoSessionParametersKHR *par = av_malloc(sizeof(*par));
1070  const FFVulkanFunctions *vk = &ctx->s.vkfn;
1071  VkResult ret;
1072 
1073  if (!par)
1074  return AVERROR(ENOMEM);
1075 
1076  /* Create session parameters */
1077  ret = vk->CreateVideoSessionParametersKHR(ctx->s.hwctx->act_dev, session_params_create,
1078  ctx->s.hwctx->alloc, par);
1079  if (ret != VK_SUCCESS) {
1080  av_log(logctx, AV_LOG_ERROR, "Unable to create Vulkan video session parameters: %s!\n",
1081  ff_vk_ret2str(ret));
1082  av_free(par);
1083  return AVERROR_EXTERNAL;
1084  }
1085  *par_ref = av_buffer_create((uint8_t *)par, sizeof(*par),
1087  if (!*par_ref) {
1088  vk_decode_free_params(ctx, (uint8_t *)par);
1089  return AVERROR(ENOMEM);
1090  }
1091 
1092  return 0;
1093 }
1094 
1096 {
1099 
1100  /* Wait on and free execution pool */
1101  ff_vk_exec_pool_free(&ctx->s, &dec->exec_pool);
1102 
1103  av_freep(&dec->hevc_headers);
1105  av_buffer_unref(&dec->shared_ref);
1106  av_freep(&dec->slice_off);
1107  return 0;
1108 }
1109 
1111 {
1112  int err, qf, cxpos = 0, cypos = 0, nb_q = 0;
1113  VkResult ret;
1116  FFVulkanContext *s;
1117  FFVulkanFunctions *vk;
1118  const VkVideoProfileInfoKHR *profile;
1119 
1120  VkVideoDecodeH264SessionParametersCreateInfoKHR h264_params = {
1121  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H264_SESSION_PARAMETERS_CREATE_INFO_KHR,
1122  };
1123  VkVideoDecodeH265SessionParametersCreateInfoKHR h265_params = {
1124  .sType = VK_STRUCTURE_TYPE_VIDEO_DECODE_H265_SESSION_PARAMETERS_CREATE_INFO_KHR,
1125  };
1128  };
1129  VkVideoSessionParametersCreateInfoKHR session_params_create = {
1130  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_PARAMETERS_CREATE_INFO_KHR,
1131  .pNext = avctx->codec_id == AV_CODEC_ID_H264 ? (void *)&h264_params :
1132  avctx->codec_id == AV_CODEC_ID_HEVC ? (void *)&h265_params :
1133  avctx->codec_id == AV_CODEC_ID_AV1 ? (void *)&av1_params :
1134  NULL,
1135  };
1136  VkVideoSessionCreateInfoKHR session_create = {
1137  .sType = VK_STRUCTURE_TYPE_VIDEO_SESSION_CREATE_INFO_KHR,
1138  };
1139  VkSamplerYcbcrConversionCreateInfo yuv_sampler_info = {
1140  .sType = VK_STRUCTURE_TYPE_SAMPLER_YCBCR_CONVERSION_CREATE_INFO,
1141  .components = ff_comp_identity_map,
1142  .ycbcrModel = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY,
1143  .ycbcrRange = avctx->color_range == AVCOL_RANGE_MPEG, /* Ignored */
1144  };
1145 
1147  if (err < 0)
1148  return err;
1149 
1150  /* Initialize contexts */
1152  s = &ctx->s;
1153  vk = &ctx->s.vkfn;
1154 
1155  s->frames_ref = av_buffer_ref(avctx->hw_frames_ctx);
1156  s->frames = (AVHWFramesContext *)s->frames_ref->data;
1157  s->hwfc = s->frames->hwctx;
1158 
1159  s->device = (AVHWDeviceContext *)s->frames->device_ref->data;
1160  s->hwctx = s->device->hwctx;
1161 
1163  if (!profile) {
1164  av_log(avctx, AV_LOG_ERROR, "Video profile missing from frames context!");
1165  return AVERROR(EINVAL);
1166  }
1167 
1168  err = ff_vk_load_props(s);
1169  if (err < 0)
1170  goto fail;
1171 
1172  /* Create queue context */
1173  qf = ff_vk_qf_init(s, &ctx->qf, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1174 
1175  /* Check for support */
1176  if (!(s->video_props[qf].videoCodecOperations &
1177  ff_vk_codec_map[avctx->codec_id].decode_op)) {
1178  av_log(avctx, AV_LOG_ERROR, "Decoding %s not supported on the given "
1179  "queue family %i!\n", avcodec_get_name(avctx->codec_id), qf);
1180  return AVERROR(EINVAL);
1181  }
1182 
1183  /* Enable queries if supported */
1184  if (s->query_props[qf].queryResultStatusSupport)
1185  nb_q = 1;
1186 
1187  session_create.flags = 0x0;
1188  session_create.queueFamilyIndex = s->hwctx->queue_family_decode_index;
1189  session_create.maxCodedExtent = ctx->caps.maxCodedExtent;
1190  session_create.maxDpbSlots = ctx->caps.maxDpbSlots;
1191  session_create.maxActiveReferencePictures = ctx->caps.maxActiveReferencePictures;
1192  session_create.pictureFormat = s->hwfc->format[0];
1193  session_create.referencePictureFormat = session_create.pictureFormat;
1194  session_create.pStdHeaderVersion = dec_ext[avctx->codec_id];
1195  session_create.pVideoProfile = profile;
1196 
1197  /* Create decode exec context for this specific main thread.
1198  * 2 async contexts per thread was experimentally determined to be optimal
1199  * for a majority of streams. */
1200  err = ff_vk_exec_pool_init(s, &ctx->qf, &dec->exec_pool, 2,
1201  nb_q, VK_QUERY_TYPE_RESULT_STATUS_ONLY_KHR, 0,
1202  profile);
1203  if (err < 0)
1204  goto fail;
1205 
1206  err = ff_vk_video_common_init(avctx, s, &ctx->common, &session_create);
1207  if (err < 0)
1208  goto fail;
1209 
1210  /* Get sampler */
1212  yuv_sampler_info.xChromaOffset = cxpos >> 7;
1213  yuv_sampler_info.yChromaOffset = cypos >> 7;
1214  yuv_sampler_info.format = s->hwfc->format[0];
1215  ret = vk->CreateSamplerYcbcrConversion(s->hwctx->act_dev, &yuv_sampler_info,
1216  s->hwctx->alloc, &ctx->yuv_sampler);
1217  if (ret != VK_SUCCESS) {
1218  err = AVERROR_EXTERNAL;
1219  goto fail;
1220  }
1221 
1222  /* If doing an out-of-place decoding, create a DPB pool */
1223  if (dec->dedicated_dpb || avctx->codec_id == AV_CODEC_ID_AV1) {
1225  AVVulkanFramesContext *dpb_hwfc;
1226 
1227  ctx->dpb_hwfc_ref = av_hwframe_ctx_alloc(s->frames->device_ref);
1228  if (!ctx->dpb_hwfc_ref) {
1229  err = AVERROR(ENOMEM);
1230  goto fail;
1231  }
1232 
1233  dpb_frames = (AVHWFramesContext *)ctx->dpb_hwfc_ref->data;
1234  dpb_frames->format = s->frames->format;
1235  dpb_frames->sw_format = s->frames->sw_format;
1236  dpb_frames->width = s->frames->width;
1237  dpb_frames->height = s->frames->height;
1238 
1239  dpb_hwfc = dpb_frames->hwctx;
1240  dpb_hwfc->create_pnext = (void *)ff_vk_find_struct(ctx->s.hwfc->create_pnext,
1241  VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
1242  dpb_hwfc->format[0] = s->hwfc->format[0];
1243  dpb_hwfc->tiling = VK_IMAGE_TILING_OPTIMAL;
1244  dpb_hwfc->usage = VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR |
1245  VK_IMAGE_USAGE_SAMPLED_BIT; /* Shuts validator up. */
1246 
1247  if (dec->layered_dpb)
1248  dpb_hwfc->nb_layers = ctx->caps.maxDpbSlots;
1249 
1250  err = av_hwframe_ctx_init(ctx->dpb_hwfc_ref);
1251  if (err < 0)
1252  goto fail;
1253 
1254  if (dec->layered_dpb) {
1255  ctx->layered_frame = vk_get_dpb_pool(ctx);
1256  if (!ctx->layered_frame) {
1257  err = AVERROR(ENOMEM);
1258  goto fail;
1259  }
1260 
1261  err = vk_decode_create_view(ctx, &ctx->layered_view, &ctx->layered_aspect,
1262  (AVVkFrame *)ctx->layered_frame->data[0],
1263  s->hwfc->format[0]);
1264  if (err < 0)
1265  goto fail;
1266  }
1267  }
1268 
1269  session_params_create.videoSession = ctx->common.session;
1270  ret = vk->CreateVideoSessionParametersKHR(s->hwctx->act_dev, &session_params_create,
1271  s->hwctx->alloc, &ctx->empty_session_params);
1272  if (ret != VK_SUCCESS) {
1273  av_log(avctx, AV_LOG_ERROR, "Unable to create empty Vulkan video session parameters: %s!\n",
1274  ff_vk_ret2str(ret));
1275  return AVERROR_EXTERNAL;
1276  }
1277 
1278  ff_vk_decode_flush(avctx);
1279 
1280  av_log(avctx, AV_LOG_VERBOSE, "Vulkan decoder initialization sucessful\n");
1281 
1282  return 0;
1283 
1284 fail:
1285  ff_vk_decode_uninit(avctx);
1286 
1287  return err;
1288 }
VkVideoDecodeAV1ProfileInfoMESA
Definition: vulkan_video_codec_av1std.h:373
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:92
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:86
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:65
FFVulkanDecodePicture::slices_size
size_t slices_size
Definition: vulkan_decode.h:99
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:494
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
AV_FIELD_PROGRESSIVE
@ AV_FIELD_PROGRESSIVE
Definition: codec_par.h:40
FFVulkanDecodeProfileData::profile
VkVideoProfileInfoKHR profile
Definition: vulkan_decode.h:33
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
AV_PROFILE_H264_INTRA
#define AV_PROFILE_H264_INTRA
Definition: defs.h:107
FFVulkanDecodeProfileData::h265_profile
VkVideoDecodeH265ProfileInfoKHR h265_profile
Definition: vulkan_decode.h:30
ff_comp_identity_map
const VkComponentMapping ff_comp_identity_map
Definition: vulkan.c:26
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2936
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
AVHWFramesContext::format
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
Definition: hwcontext.h:209
FFVulkanDecodeContext::exec_pool
FFVkExecPool exec_pool
Definition: vulkan_decode.h:59
dpb_frames
int dpb_frames
Definition: h264_levels.c:163
ff_vk_qf_init
int ff_vk_qf_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, VkQueueFlagBits dev_family)
Chooses a QF and loads it into a context.
Definition: vulkan.c:225
FFVkVideoBuffer
Definition: vulkan_video.h:81
AV_PROFILE_HEVC_MAIN
#define AV_PROFILE_HEVC_MAIN
Definition: defs.h:158
ff_vk_exec_update_frame
void ff_vk_exec_update_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkImageMemoryBarrier2 *bar, uint32_t *nb_img_bar)
Definition: vulkan.c:676
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:100
FFVkVideoBuffer::buf
FFVkBuffer buf
Definition: vulkan_video.h:82
av_hwframe_ctx_init
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
Definition: hwcontext.c:334
ff_vk_video_common_init
av_cold int ff_vk_video_common_init(void *log, FFVulkanContext *s, FFVkVideoCommon *common, VkVideoSessionCreateInfoKHR *session_create)
Initialize video session, allocating and binding necessary memory.
Definition: vulkan_video.c:302
ff_vk_depth_from_av_depth
VkVideoComponentBitDepthFlagBitsKHR ff_vk_depth_from_av_depth(int depth)
Get Vulkan's bit depth from an [8:12] integer.
Definition: vulkan_video.c:146
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
VkVideoDecodeAV1ProfileInfoMESA::stdProfileIdc
StdVideoAV1MESAProfile stdProfileIdc
Definition: vulkan_video_codec_av1std.h:376
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
av_hwframe_ctx_alloc
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
Definition: hwcontext.c:248
AVHWFramesContext::free
void(* free)(struct AVHWFramesContext *ctx)
This field may be set by the caller before calling av_hwframe_ctx_init().
Definition: hwcontext.h:170
AVCodecContext::field_order
enum AVFieldOrder field_order
Field order.
Definition: avcodec.h:1057
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:206
b
#define b
Definition: input.c:41
data
const char data[16]
Definition: mxf.c:148
FFVulkanDecodeContext::frame_id_alloc_mask
uint32_t frame_id_alloc_mask
Definition: vulkan_decode.h:64
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:196
VkVideoDecodeAV1CapabilitiesMESA::flags
VkVideoDecodeAV1CapabilityFlagsMESA flags
Definition: vulkan_video_codec_av1std.h:388
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:1892
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
VkVideoDecodeAV1SessionParametersCreateInfoMESA
Definition: vulkan_video_codec_av1std.h:367
avcodec_profile_name
const char * avcodec_profile_name(enum AVCodecID codec_id, int profile)
Return a name for the specified profile, if available.
Definition: utils.c:447
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:229
AVFrame::buf
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:590
ff_vk_dec_hevc_ext
const VkExtensionProperties ff_vk_dec_hevc_ext
Definition: vulkan_hevc.c:25
FFVulkanDecodeContext
Definition: vulkan_decode.h:56
FFVulkanDecodeProfileData::av1_profile
VkVideoDecodeAV1ProfileInfoMESA av1_profile
Definition: vulkan_decode.h:31
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:376
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:596
ff_vk_decode_prepare_frame
int ff_vk_decode_prepare_frame(FFVulkanDecodeContext *dec, AVFrame *pic, FFVulkanDecodePicture *vkpic, int is_current, int alloc_dpb)
Prepare a frame, creates the image view, and sets up the dpb fields.
Definition: vulkan_decode.c:173
AV_HWACCEL_FLAG_IGNORE_LEVEL
#define AV_HWACCEL_FLAG_IGNORE_LEVEL
Hardware acceleration should be used for decoding even if the codec level used is unknown or higher t...
Definition: avcodec.h:2173
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:361
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
FFVulkanDecodeContext::session_params
AVBufferRef * session_params
Definition: vulkan_decode.h:58
ff_vk_subsampling_from_av_desc
VkVideoChromaSubsamplingFlagBitsKHR ff_vk_subsampling_from_av_desc(const AVPixFmtDescriptor *desc)
Get Vulkan's chroma subsampling from a pixfmt descriptor.
Definition: vulkan_video.c:133
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:48
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:270
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:279
FF_VK_EXT_VIDEO_DECODE_QUEUE
@ FF_VK_EXT_VIDEO_DECODE_QUEUE
Definition: vulkan_functions.h:43
fail
#define fail()
Definition: checkasm.h:138
FFVulkanDecodePicture::sem_value
uint64_t sem_value
Definition: vulkan_decode.h:84
frames
if it could not because there are no more frames
Definition: filter_design.txt:266
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:176
AV_FIELD_UNKNOWN
@ AV_FIELD_UNKNOWN
Definition: codec_par.h:39
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:34
ff_vk_decode_frame
int ff_vk_decode_frame(AVCodecContext *avctx, AVFrame *pic, FFVulkanDecodePicture *vp, AVFrame *rpic[], FFVulkanDecodePicture *rvkp[])
Decode a frame.
Definition: vulkan_decode.c:329
FFVulkanDecodeShared
Definition: vulkan_decode.h:37
vulkan_decode_get_profile
static int vulkan_decode_get_profile(AVCodecContext *avctx, AVBufferRef *frames_ref, enum AVPixelFormat *pix_fmt, VkFormat *vk_fmt, FFVulkanDecodeProfileData *prof, int *dpb_dedicate)
Definition: vulkan_decode.c:738
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:61
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:88
VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_MESA
#define VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_SESSION_PARAMETERS_CREATE_INFO_MESA
Definition: vulkan_video_codec_av1std.h:393
FFVulkanDecodeContext::slice_off
uint32_t * slice_off
Definition: vulkan_decode.h:70
FFVulkanDecodePicture::img_view_out
VkImageView img_view_out
Definition: vulkan_decode.h:78
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
FFVulkanDecodeProfileData::h264_profile
VkVideoDecodeH264ProfileInfoKHR h264_profile
Definition: vulkan_decode.h:29
VkVideoDecodeAV1SessionParametersCreateInfoMESA::sType
VkStructureType sType
Definition: vulkan_video_codec_av1std.h:368
AVHWFramesContext::height
int height
Definition: hwcontext.h:229
FFVulkanDecodeContext::shared_ref
AVBufferRef * shared_ref
Definition: vulkan_decode.h:57
FFVkCodecMap::decode_op
VkVideoCodecOperationFlagBitsKHR decode_op
Definition: vulkan_video.h:38
VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_MESA
#define VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_CAPABILITIES_MESA
Definition: vulkan_video_codec_av1std.h:396
FFVulkanDecodeProfileData::profile_list
VkVideoProfileListInfoKHR profile_list
Definition: vulkan_decode.h:34
av_fast_realloc
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given buffer if it is not large enough, otherwise do nothing.
Definition: mem.c:495
ref_frame
static const AVFrame * ref_frame(const struct pl_frame_mix *mix)
Definition: vf_libplacebo.c:783
FFVulkanDecodePicture::wait_semaphores
PFN_vkWaitSemaphores wait_semaphores
Definition: vulkan_decode.h:102
s
#define s(width, name)
Definition: cbs_vp9.c:198
FFVulkanDecodePicture
Definition: vulkan_decode.h:74
av_chroma_location_enum_to_pos
int av_chroma_location_enum_to_pos(int *xpos, int *ypos, enum AVChromaLocation pos)
Converts AVChromaLocation to swscale x/y chroma position.
Definition: pixdesc.c:3354
ff_vk_decode_create_params
int ff_vk_decode_create_params(AVBufferRef **par_ref, void *logctx, FFVulkanDecodeShared *ctx, const VkVideoSessionParametersCreateInfoKHR *session_params_create)
Create VkVideoSessionParametersKHR wrapped in an AVBufferRef.
Definition: vulkan_decode.c:1066
ff_vk_exec_mirror_sem_value
int ff_vk_exec_mirror_sem_value(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *dst, uint64_t *dst_val, AVFrame *f)
Definition: vulkan.c:695
offsets
static const int offsets[]
Definition: hevc_pel.c:34
AVFormatContext::flags
int flags
Flags modifying the (de)muxer behaviour.
Definition: avformat.h:1223
format
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate. The lists are not just lists
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:89
pix_fmt
static enum AVPixelFormat pix_fmt
Definition: demux_decode.c:41
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:501
vulkan_setup_profile
static VkResult vulkan_setup_profile(AVCodecContext *avctx, FFVulkanDecodeProfileData *prof, AVVulkanDeviceContext *hwctx, FFVulkanFunctions *vk, const struct FFVkCodecMap *vk_codec, VkVideoDecodeH264CapabilitiesKHR *h264_caps, VkVideoDecodeH265CapabilitiesKHR *h265_caps, VkVideoDecodeAV1CapabilitiesMESA *av1_caps, VkVideoCapabilitiesKHR *caps, VkVideoDecodeCapabilitiesKHR *dec_caps, int cur_profile)
Definition: vulkan_decode.c:664
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:572
codec_id
enum AVCodecID codec_id
Definition: vaapi_decode.c:388
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:256
AV_CODEC_ID_H264
@ AV_CODEC_ID_H264
Definition: codec_id.h:79
VkVideoDecodeAV1CapabilitiesMESA
Definition: vulkan_video_codec_av1std.h:385
FFVkExecContext::query_idx
int query_idx
Definition: vulkan.h:170
ff_decode_get_hw_frames_ctx
int ff_decode_get_hw_frames_ctx(AVCodecContext *avctx, enum AVHWDeviceType dev_type)
Make sure avctx.hw_frames_ctx is set.
Definition: decode.c:1109
AVCodecContext::codec_id
enum AVCodecID codec_id
Definition: avcodec.h:447
if
if(ret)
Definition: filter_design.txt:179
VkVideoDecodeAV1ProfileInfoMESA::sType
VkStructureType sType
Definition: vulkan_video_codec_av1std.h:374
get_video_profile
static const VkVideoProfileInfoKHR * get_video_profile(FFVulkanDecodeShared *ctx, enum AVCodecID codec_id)
Definition: vulkan_decode.c:45
STD_VIDEO_AV1_MESA_PROFILE_MAIN
@ STD_VIDEO_AV1_MESA_PROFILE_MAIN
Definition: vulkan_video_codec_av1std.h:37
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:44
FFVulkanDecodePicture::img_view_ref
VkImageView img_view_ref
Definition: vulkan_decode.h:77
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:222
AVCodecContext::color_range
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: avcodec.h:1035
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AV_CODEC_ID_AV1
@ AV_CODEC_ID_AV1
Definition: codec_id.h:283
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:99
ff_vk_decode_free_frame
void ff_vk_decode_free_frame(AVHWDeviceContext *dev_ctx, FFVulkanDecodePicture *vp)
Free a frame and its state.
Definition: vulkan_decode.c:558
ff_vk_video_common_uninit
av_cold void ff_vk_video_common_uninit(FFVulkanContext *s, FFVkVideoCommon *common)
Free video session and required resources.
Definition: vulkan_video.c:282
AVCodecContext::internal
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:472
FFVulkanDecodeProfileData::usage
VkVideoDecodeUsageInfoKHR usage
Definition: vulkan_decode.h:32
AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH
#define AV_HWACCEL_FLAG_ALLOW_PROFILE_MISMATCH
Hardware acceleration should still be attempted for decoding when the codec profile does not match th...
Definition: avcodec.h:2193
ff_vk_decode_uninit
int ff_vk_decode_uninit(AVCodecContext *avctx)
Free decoder.
Definition: vulkan_decode.c:1095
FFVkVideoBuffer::mem
uint8_t * mem
Definition: vulkan_video.h:83
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:236
FFVkBuffer::size
size_t size
Definition: vulkan.h:100
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:195
FFVkExecPool::nb_queries
int nb_queries
Definition: vulkan.h:224
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:296
FFVulkanContext
Definition: vulkan.h:228
ff_vk_frame_params
int ff_vk_frame_params(AVCodecContext *avctx, AVBufferRef *hw_frames_ctx)
Initialize hw_frames_ctx with the parameters needed to decode the stream using the parameters from av...
Definition: vulkan_decode.c:1009
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:49
usage
const char * usage
Definition: floatimg_cmp.c:60
VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_MESA
#define VK_STRUCTURE_TYPE_VIDEO_DECODE_AV1_PROFILE_INFO_MESA
Definition: vulkan_video_codec_av1std.h:397
ff_vk_h265_level_to_av
int ff_vk_h265_level_to_av(StdVideoH265LevelIdc level)
Definition: vulkan_video.c:183
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AVCodecInternal::hwaccel_priv_data
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:124
AVVkFrame
Definition: hwcontext_vulkan.h:265
ff_vk_dec_h264_ext
const VkExtensionProperties ff_vk_dec_h264_ext
Definition: vulkan_h264.c:24
size
int size
Definition: twinvq_data.h:10344
FFVkCodecMap
Definition: vulkan_video.h:34
FFVulkanDecodePicture::img_aspect_ref
VkImageAspectFlags img_aspect_ref
Definition: vulkan_decode.h:81
VkVideoDecodeAV1CapabilitiesMESA::sType
VkStructureType sType
Definition: vulkan_video_codec_av1std.h:386
AVVkFrame::queue_family
uint32_t queue_family[AV_NUM_DATA_POINTERS]
Queue family of the images.
Definition: hwcontext_vulkan.h:329
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
FFVkExecContext
Definition: vulkan.h:153
vk_decode_free_params
static void vk_decode_free_params(void *opaque, uint8_t *data)
Definition: vulkan_decode.c:1056
ff_vk_codec_map
const FFVkCodecMap ff_vk_codec_map[AV_CODEC_ID_FIRST_AUDIO]
Index is codec_id.
Definition: vulkan_video.c:23
FFVulkanDecodeProfileData
Definition: vulkan_decode.h:28
dec_ext
static const VkExtensionProperties * dec_ext[]
Definition: vulkan_decode.c:33
avcodec_get_name
const char * avcodec_get_name(enum AVCodecID id)
Get the name of a codec.
Definition: utils.c:414
ff_vk_exec_start
int ff_vk_exec_start(FFVulkanContext *s, FFVkExecContext *e)
Start/submit/wait an execution.
Definition: vulkan.c:510
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:244
VkFormat
enum VkFormat VkFormat
Definition: hwcontext_stub.c:25
ff_vk_h264_level_to_av
int ff_vk_h264_level_to_av(StdVideoH264LevelIdc level)
Convert level from Vulkan to AV.
Definition: vulkan_video.c:157
AV_CODEC_ID_HEVC
@ AV_CODEC_ID_HEVC
Definition: codec_id.h:226
FFVulkanDecodePicture::sem
VkSemaphore sem
Definition: vulkan_decode.h:83
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:254
FFVulkanDecodeContext::external_fg
int external_fg
Definition: vulkan_decode.h:63
av_buffer_replace
int av_buffer_replace(AVBufferRef **pdst, const AVBufferRef *src)
Ensure dst refers to the same data as src.
Definition: buffer.c:233
AVCodecContext::chroma_sample_location
enum AVChromaLocation chroma_sample_location
This defines the location of chroma samples.
Definition: avcodec.h:1042
profile
int profile
Definition: mxfenc.c:2111
ff_vk_exec_get_query
VkResult ff_vk_exec_get_query(FFVulkanContext *s, FFVkExecContext *e, void **data, int64_t *status)
Performs nb_queries queries and returns their results and statuses.
Definition: vulkan.c:447
AVCodecContext::height
int height
Definition: avcodec.h:617
AVCOL_RANGE_MPEG
@ AVCOL_RANGE_MPEG
Narrow or limited range content.
Definition: pixfmt.h:652
AVCodecContext::hw_frames_ctx
AVBufferRef * hw_frames_ctx
A reference to the AVHWFramesContext describing the input (for encoding) or output (decoding) frames.
Definition: avcodec.h:1934
CODEC_VER
#define CODEC_VER(ver)
Definition: vulkan_video.h:32
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:124
ff_vk_decode_flush
void ff_vk_decode_flush(AVCodecContext *avctx)
Flush decoder.
Definition: vulkan_decode.c:299
ret
ret
Definition: filter_design.txt:187
VK_VIDEO_DECODE_AV1_CAPABILITY_EXTERNAL_FILM_GRAIN_MESA
@ VK_VIDEO_DECODE_AV1_CAPABILITY_EXTERNAL_FILM_GRAIN_MESA
Definition: vulkan_video_codec_av1std.h:380
AVHWFramesContext::hwctx
void * hwctx
The format-specific data, allocated and freed automatically along with this context.
Definition: hwcontext.h:162
AVHWFramesContext::user_opaque
void * user_opaque
Arbitrary user data, to be used e.g.
Definition: hwcontext.h:175
ff_vk_decode_add_slice
int ff_vk_decode_add_slice(AVCodecContext *avctx, FFVulkanDecodePicture *vp, const uint8_t *data, size_t size, int add_startcode, uint32_t *nb_slices, const uint32_t **offsets)
Add slice data to frame.
Definition: vulkan_decode.c:237
AV_PROFILE_H264_CONSTRAINED
#define AV_PROFILE_H264_CONSTRAINED
Definition: defs.h:106
FFVulkanDecodeContext::hevc_headers
struct HEVCHeaderSet * hevc_headers
Definition: vulkan_decode.h:67
FFVkExecContext::buf
VkCommandBuffer buf
Definition: vulkan.h:164
AVFrame::hw_frames_ctx
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
Definition: frame.h:752
AVCodecContext
main external API structure.
Definition: avcodec.h:437
FFVulkanDecodeContext::dedicated_dpb
int dedicated_dpb
Definition: vulkan_decode.h:61
av_find_best_pix_fmt_of_2
enum AVPixelFormat av_find_best_pix_fmt_of_2(enum AVPixelFormat dst_pix_fmt1, enum AVPixelFormat dst_pix_fmt2, enum AVPixelFormat src_pix_fmt, int has_alpha, int *loss_ptr)
Compute what kind of losses will occur when converting from one specific pixel format to another.
Definition: pixdesc.c:3215
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
AV_CODEC_ID_H265
#define AV_CODEC_ID_H265
Definition: codec_id.h:227
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
ff_vk_params_invalidate
int ff_vk_params_invalidate(AVCodecContext *avctx, int t, const uint8_t *b, uint32_t s)
Removes current session parameters to recreate them.
Definition: vulkan_decode.c:110
ff_vk_video_get_buffer
int ff_vk_video_get_buffer(FFVulkanContext *ctx, FFVkVideoCommon *s, AVBufferRef **buf, VkBufferUsageFlags usage, void *create_pNext, size_t size)
Get a mapped FFVkPooledBuffer with a specific guaranteed minimum size from a pool.
Definition: vulkan_video.c:224
FFVulkanDecodePicture::dpb_frame
AVFrame * dpb_frame
Definition: vulkan_decode.h:75
AV_PROFILE_H264_CONSTRAINED_BASELINE
#define AV_PROFILE_H264_CONSTRAINED_BASELINE
Definition: defs.h:110
ff_vk_update_thread_context
int ff_vk_update_thread_context(AVCodecContext *dst, const AVCodecContext *src)
Synchronize the contexts between 2 threads.
Definition: vulkan_decode.c:67
AVVulkanFramesContext::tiling
VkImageTiling tiling
Controls the tiling of allocated frames.
Definition: hwcontext_vulkan.h:185
vulkan_video.h
desc
const char * desc
Definition: libsvtav1.c:83
AVVulkanDeviceContext::enabled_dev_extensions
const char *const * enabled_dev_extensions
Enabled device extensions.
Definition: hwcontext_vulkan.h:98
AVVulkanFramesContext::nb_layers
int nb_layers
Number of layers each image will have.
Definition: hwcontext_vulkan.h:241
FFVulkanDecodePicture::slices_buf
AVBufferRef * slices_buf
Definition: vulkan_decode.h:98
AVVkFrame::layout
VkImageLayout layout[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:295
AVBufferRef
A reference to a data buffer.
Definition: buffer.h:82
FFVkExecPool::cmd_bufs
VkCommandBuffer * cmd_bufs
Definition: vulkan.h:215
ff_vk_dec_av1_ext
const VkExtensionProperties ff_vk_dec_av1_ext
Definition: vulkan_av1.c:26
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:70
vulkan_decode.h
free_common
static void free_common(void *opaque, uint8_t *data)
Definition: vulkan_decode.c:589
FFVulkanDecodePicture::img_view_dest
VkImageView img_view_dest
Definition: vulkan_decode.h:79
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
decode_end
static av_cold int decode_end(AVCodecContext *avctx)
Definition: 4xm.c:975
vk_decode_create_view
static int vk_decode_create_view(FFVulkanDecodeShared *ctx, VkImageView *dst_view, VkImageAspectFlags *aspect, AVVkFrame *src, VkFormat vkf)
Definition: vulkan_decode.c:117
FFALIGN
#define FFALIGN(x, a)
Definition: macros.h:78
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
FFVulkanDecodeContext::layered_dpb
int layered_dpb
Definition: vulkan_decode.h:62
AVCodecContext::width
int width
picture width / height.
Definition: avcodec.h:617
vk_get_dpb_pool
static AVFrame * vk_get_dpb_pool(FFVulkanDecodeShared *ctx)
Definition: vulkan_decode.c:159
ff_vk_exec_submit
int ff_vk_exec_submit(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:721
FFVulkanDecodePicture::destroy_image_view
PFN_vkDestroyImageView destroy_image_view
Definition: vulkan_decode.h:103
ff_vk_extensions_to_mask
static uint64_t ff_vk_extensions_to_mask(const char *const *extensions, int nb_extensions)
Definition: vulkan_loader.h:34
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
vulkan_decode_bootstrap
static int vulkan_decode_bootstrap(AVCodecContext *avctx, AVBufferRef *frames_ref)
Definition: vulkan_decode.c:620
ff_vk_pix_fmt_from_vkfmt
enum AVPixelFormat ff_vk_pix_fmt_from_vkfmt(VkFormat vkf)
Get pixfmt from a Vulkan format.
Definition: vulkan_video.c:117
ff_vk_decode_init
int ff_vk_decode_init(AVCodecContext *avctx)
Initialize decoder.
Definition: vulkan_decode.c:1110
ff_vk_aspect_bits_from_vkfmt
VkImageAspectFlags ff_vk_aspect_bits_from_vkfmt(VkFormat vkf)
Get aspect bits which include all planes from a VkFormat.
Definition: vulkan_video.c:125
FFVulkanDecodePicture::decode_info
VkVideoDecodeInfoKHR decode_info
Definition: vulkan_decode.h:95
AVCodecContext::sw_pix_fmt
enum AVPixelFormat sw_pix_fmt
Nominal unaccelerated pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:1804
av_hwframe_get_buffer
int av_hwframe_get_buffer(AVBufferRef *hwframe_ref, AVFrame *frame, int flags)
Allocate a new frame attached to the given AVHWFramesContext.
Definition: hwcontext.c:507
FFVulkanFunctions
Definition: vulkan_functions.h:226
FFVkExecPool::pool_size
int pool_size
Definition: vulkan.h:216
free_profile_data
static void free_profile_data(AVHWFramesContext *hwfc)
Definition: vulkan_decode.c:1004
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2856
FFVulkanDecodePicture::img_aspect
VkImageAspectFlags img_aspect
Definition: vulkan_decode.h:80