FFmpeg
hwcontext_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #define VK_NO_PROTOTYPES
22 #define VK_ENABLE_BETA_EXTENSIONS
23 
24 #ifdef _WIN32
25 #include <windows.h> /* Included to prevent conflicts with CreateSemaphore */
26 #include <versionhelpers.h>
27 #include "compat/w32dlfcn.h"
28 #else
29 #include <dlfcn.h>
30 #include <unistd.h>
31 #endif
32 
33 #include "thread.h"
34 
35 #include "config.h"
36 #include "pixdesc.h"
37 #include "avstring.h"
38 #include "imgutils.h"
39 #include "hwcontext.h"
40 #include "hwcontext_internal.h"
41 #include "hwcontext_vulkan.h"
42 #include "mem.h"
43 
44 #include "vulkan.h"
45 #include "vulkan_loader.h"
46 
47 #if CONFIG_VAAPI
48 #include "hwcontext_vaapi.h"
49 #endif
50 
51 #if CONFIG_LIBDRM
52 #if CONFIG_VAAPI
53 #include <va/va_drmcommon.h>
54 #endif
55 #ifdef __linux__
56 #include <sys/sysmacros.h>
57 #endif
58 #include <sys/stat.h>
59 #include <xf86drm.h>
60 #include <drm_fourcc.h>
61 #include "hwcontext_drm.h"
62 #endif
63 
64 #if HAVE_LINUX_DMA_BUF_H
65 #include <sys/ioctl.h>
66 #include <linux/dma-buf.h>
67 #endif
68 
69 #if CONFIG_CUDA
70 #include "hwcontext_cuda_internal.h"
71 #include "cuda_check.h"
72 #define CHECK_CU(x) FF_CUDA_CHECK_DL(cuda_cu, cu, x)
73 #endif
74 
75 typedef struct VulkanDeviceFeatures {
76  VkPhysicalDeviceFeatures2 device;
77 
78  VkPhysicalDeviceVulkan11Features vulkan_1_1;
79  VkPhysicalDeviceVulkan12Features vulkan_1_2;
80  VkPhysicalDeviceVulkan13Features vulkan_1_3;
81  VkPhysicalDeviceTimelineSemaphoreFeatures timeline_semaphore;
82 
83  VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1;
84 
85  VkPhysicalDeviceShaderObjectFeaturesEXT shader_object;
86  VkPhysicalDeviceCooperativeMatrixFeaturesKHR cooperative_matrix;
87  VkPhysicalDeviceDescriptorBufferFeaturesEXT descriptor_buffer;
88  VkPhysicalDeviceShaderAtomicFloatFeaturesEXT atomic_float;
89 
90  VkPhysicalDeviceOpticalFlowFeaturesNV optical_flow;
91 
92 #ifdef VK_KHR_shader_relaxed_extended_instruction
93  VkPhysicalDeviceShaderRelaxedExtendedInstructionFeaturesKHR relaxed_extended_instruction;
94 #endif
95 } VulkanDeviceFeatures;
96 
97 typedef struct VulkanDevicePriv {
98  /**
99  * The public AVVulkanDeviceContext. See hwcontext_vulkan.h for it.
100  */
101  AVVulkanDeviceContext p;
102 
103  /* Vulkan library and loader functions */
104  void *libvulkan;
105 
106  FFVulkanContext vkctx;
107  FFVkQueueFamilyCtx compute_qf;
108  FFVkQueueFamilyCtx transfer_qf;
109 
110  /* Properties */
111  VkPhysicalDeviceProperties2 props;
112  VkPhysicalDeviceMemoryProperties mprops;
113  VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops;
114 
115  /* Enabled features */
116  VulkanDeviceFeatures feats;
117 
118  /* Queues */
119  pthread_mutex_t **qf_mutex;
120  uint32_t nb_tot_qfs;
121  uint32_t img_qfs[5];
122  uint32_t nb_img_qfs;
123 
124  /* Debug callback */
125  VkDebugUtilsMessengerEXT debug_ctx;
126 
127  /* Settings */
128  int use_linear_images;
129 
130  /* Option to allocate all image planes in a single allocation */
131  int contiguous_planes;
132 
133  /* Disable multiplane images */
134  int disable_multiplane;
135 
136  /* Nvidia */
137  int dev_is_nvidia;
138 } VulkanDevicePriv;
139 
140 typedef struct VulkanFramesPriv {
141  /**
142  * The public AVVulkanFramesContext. See hwcontext_vulkan.h for it.
143  */
144  AVVulkanFramesContext p;
145 
146  /* Image conversions */
147  FFVkExecPool compute_exec;
148 
149  /* Image transfers */
150  FFVkExecPool upload_exec;
151  FFVkExecPool download_exec;
152 
153  /* Temporary buffer pools */
154  AVBufferPool *tmp;
155 
156  /* Modifier info list to free at uninit */
157  VkImageDrmFormatModifierListCreateInfoEXT *modifier_info;
158 } VulkanFramesPriv;
159 
160 typedef struct AVVkFrameInternal {
161  pthread_mutex_t update_mutex;
162 
163 #if CONFIG_CUDA
164  /* Importing external memory into cuda is really expensive so we keep the
165  * memory imported all the time */
166  AVBufferRef *cuda_fc_ref; /* Need to keep it around for uninit */
167  CUexternalMemory ext_mem[AV_NUM_DATA_POINTERS];
168  CUmipmappedArray cu_mma[AV_NUM_DATA_POINTERS];
169  CUarray cu_array[AV_NUM_DATA_POINTERS];
170  CUexternalSemaphore cu_sem[AV_NUM_DATA_POINTERS];
171 #ifdef _WIN32
172  HANDLE ext_mem_handle[AV_NUM_DATA_POINTERS];
173  HANDLE ext_sem_handle[AV_NUM_DATA_POINTERS];
174 #endif
175 #endif
176 } AVVkFrameInternal;
177 
178 /* Initialize all structs in VulkanDeviceFeatures */
179 static void device_features_init(AVHWDeviceContext *ctx, VulkanDeviceFeatures *feats)
180 {
181  VulkanDevicePriv *p = ctx->hwctx;
182 
183 #define OPT_CHAIN(STRUCT_P, EXT_FLAG, TYPE) \
184  do { \
185  if ((EXT_FLAG == FF_VK_EXT_NO_FLAG) || \
186  (p->vkctx.extensions & EXT_FLAG)) { \
187  (STRUCT_P)->sType = TYPE; \
188  ff_vk_link_struct(&feats->device, STRUCT_P); \
189  } \
190  } while (0)
191 
192  feats->device = (VkPhysicalDeviceFeatures2) {
193  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2,
194  };
195 
196  OPT_CHAIN(&feats->vulkan_1_1, FF_VK_EXT_NO_FLAG,
197  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_1_FEATURES);
198  OPT_CHAIN(&feats->vulkan_1_2, FF_VK_EXT_NO_FLAG,
199  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_2_FEATURES);
200  OPT_CHAIN(&feats->vulkan_1_3, FF_VK_EXT_NO_FLAG,
201  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VULKAN_1_3_FEATURES);
202 
203  OPT_CHAIN(&feats->timeline_semaphore, FF_VK_EXT_PORTABILITY_SUBSET,
204  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_TIMELINE_SEMAPHORE_FEATURES);
205 
206  OPT_CHAIN(&feats->video_maintenance_1, FF_VK_EXT_VIDEO_MAINTENANCE_1,
207  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_VIDEO_MAINTENANCE_1_FEATURES_KHR);
208 
209  OPT_CHAIN(&feats->shader_object, FF_VK_EXT_SHADER_OBJECT,
210  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_OBJECT_FEATURES_EXT);
211  OPT_CHAIN(&feats->cooperative_matrix, FF_VK_EXT_COOP_MATRIX,
212  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_COOPERATIVE_MATRIX_FEATURES_KHR);
213  OPT_CHAIN(&feats->descriptor_buffer, FF_VK_EXT_DESCRIPTOR_BUFFER,
214  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DESCRIPTOR_BUFFER_FEATURES_EXT);
215  OPT_CHAIN(&feats->atomic_float, FF_VK_EXT_ATOMIC_FLOAT,
216  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_ATOMIC_FLOAT_FEATURES_EXT);
217 
218 #ifdef VK_KHR_shader_relaxed_extended_instruction
219  OPT_CHAIN(&feats->relaxed_extended_instruction, FF_VK_EXT_RELAXED_EXTENDED_INSTR,
220  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_SHADER_RELAXED_EXTENDED_INSTRUCTION_FEATURES_KHR);
221 #endif
222 
223  OPT_CHAIN(&feats->optical_flow, FF_VK_EXT_OPTICAL_FLOW,
224  VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_OPTICAL_FLOW_FEATURES_NV);
225 #undef OPT_CHAIN
226 }
227 
228 /* Copy all needed device features */
229 static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
230 {
231 #define COPY_VAL(VAL) \
232  do { \
233  dst->VAL = src->VAL; \
234  } while (0) \
235 
236  COPY_VAL(device.features.shaderImageGatherExtended);
237  COPY_VAL(device.features.shaderStorageImageReadWithoutFormat);
238  COPY_VAL(device.features.shaderStorageImageWriteWithoutFormat);
239  COPY_VAL(device.features.fragmentStoresAndAtomics);
240  COPY_VAL(device.features.vertexPipelineStoresAndAtomics);
241  COPY_VAL(device.features.shaderInt64);
242  COPY_VAL(device.features.shaderInt16);
243  COPY_VAL(device.features.shaderFloat64);
244 
245  COPY_VAL(vulkan_1_1.samplerYcbcrConversion);
246  COPY_VAL(vulkan_1_1.storagePushConstant16);
247  COPY_VAL(vulkan_1_1.storageBuffer16BitAccess);
248  COPY_VAL(vulkan_1_1.uniformAndStorageBuffer16BitAccess);
249 
250  COPY_VAL(vulkan_1_2.timelineSemaphore);
251  COPY_VAL(vulkan_1_2.scalarBlockLayout);
252  COPY_VAL(vulkan_1_2.bufferDeviceAddress);
253  COPY_VAL(vulkan_1_2.hostQueryReset);
254  COPY_VAL(vulkan_1_2.storagePushConstant8);
255  COPY_VAL(vulkan_1_2.shaderInt8);
256  COPY_VAL(vulkan_1_2.storageBuffer8BitAccess);
257  COPY_VAL(vulkan_1_2.uniformAndStorageBuffer8BitAccess);
258  COPY_VAL(vulkan_1_2.shaderFloat16);
259  COPY_VAL(vulkan_1_2.shaderBufferInt64Atomics);
260  COPY_VAL(vulkan_1_2.shaderSharedInt64Atomics);
261  COPY_VAL(vulkan_1_2.vulkanMemoryModel);
262  COPY_VAL(vulkan_1_2.vulkanMemoryModelDeviceScope);
263 
264  COPY_VAL(vulkan_1_3.dynamicRendering);
265  COPY_VAL(vulkan_1_3.maintenance4);
266  COPY_VAL(vulkan_1_3.synchronization2);
267  COPY_VAL(vulkan_1_3.computeFullSubgroups);
268  COPY_VAL(vulkan_1_3.shaderZeroInitializeWorkgroupMemory);
269  COPY_VAL(vulkan_1_3.dynamicRendering);
270 
271  COPY_VAL(timeline_semaphore.timelineSemaphore);
272 
273  COPY_VAL(video_maintenance_1.videoMaintenance1);
274 
275  COPY_VAL(shader_object.shaderObject);
276 
277  COPY_VAL(cooperative_matrix.cooperativeMatrix);
278 
279  COPY_VAL(descriptor_buffer.descriptorBuffer);
280  COPY_VAL(descriptor_buffer.descriptorBufferPushDescriptors);
281 
282  COPY_VAL(atomic_float.shaderBufferFloat32Atomics);
283  COPY_VAL(atomic_float.shaderBufferFloat32AtomicAdd);
284 
285 #ifdef VK_KHR_shader_relaxed_extended_instruction
286  COPY_VAL(relaxed_extended_instruction.shaderRelaxedExtendedInstruction);
287 #endif
288 
289  COPY_VAL(optical_flow.opticalFlow);
290 #undef COPY_VAL
291 }
292 
293 #define ASPECT_2PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT)
294 #define ASPECT_3PLANE (VK_IMAGE_ASPECT_PLANE_0_BIT | VK_IMAGE_ASPECT_PLANE_1_BIT | VK_IMAGE_ASPECT_PLANE_2_BIT)
295 
296 static const struct FFVkFormatEntry {
297  VkFormat vkf;
298  enum AVPixelFormat pixfmt;
299  VkImageAspectFlags aspect;
300  int vk_planes;
301  int nb_images;
302  int nb_images_fallback;
303  const VkFormat fallback[5];
304 } vk_formats_list[] = {
305  /* Gray formats */
306  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GRAY8, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8_UNORM } },
307  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
308  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY12, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
309  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY14, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
310  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GRAY16, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16_UNORM } },
311  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GRAYF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32_SFLOAT } },
312 
313  /* RGB formats */
314  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGRA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
315  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGBA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
316  { VK_FORMAT_R8G8B8_UNORM, AV_PIX_FMT_RGB24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8_UNORM } },
317  { VK_FORMAT_B8G8R8_UNORM, AV_PIX_FMT_BGR24, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8_UNORM } },
318  { VK_FORMAT_R16G16B16_UNORM, AV_PIX_FMT_RGB48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16_UNORM } },
319  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_RGBA64, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
320  { VK_FORMAT_R5G6B5_UNORM_PACK16, AV_PIX_FMT_RGB565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R5G6B5_UNORM_PACK16 } },
321  { VK_FORMAT_B5G6R5_UNORM_PACK16, AV_PIX_FMT_BGR565, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B5G6R5_UNORM_PACK16 } },
322  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_BGR0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
323  { VK_FORMAT_R8G8B8A8_UNORM, AV_PIX_FMT_RGB0, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
324  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_X2RGB10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2R10G10B10_UNORM_PACK32 } },
325  { VK_FORMAT_A2B10G10R10_UNORM_PACK32, AV_PIX_FMT_X2BGR10, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_A2B10G10R10_UNORM_PACK32 } },
326  { VK_FORMAT_R32G32B32_SFLOAT, AV_PIX_FMT_RGBF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_SFLOAT } },
327  { VK_FORMAT_R32G32B32A32_SFLOAT, AV_PIX_FMT_RGBAF32, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_SFLOAT } },
328  { VK_FORMAT_R32G32B32_UINT, AV_PIX_FMT_RGB96, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32_UINT } },
329  { VK_FORMAT_R32G32B32A32_UINT, AV_PIX_FMT_RGBA128, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R32G32B32A32_UINT } },
330 
331  /* Planar RGB */
332  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP10, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
333  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP12, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
334  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP14, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
335  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRP16, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
336  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRPF32, VK_IMAGE_ASPECT_COLOR_BIT, 3, 3, 3, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
337 
338  /* Planar RGB + Alpha */
339  { VK_FORMAT_R8_UNORM, AV_PIX_FMT_GBRAP, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
340  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP10, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
341  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP12, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
342  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP14, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
343  { VK_FORMAT_R16_UNORM, AV_PIX_FMT_GBRAP16, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
344  { VK_FORMAT_R32_SFLOAT, AV_PIX_FMT_GBRAPF32, VK_IMAGE_ASPECT_COLOR_BIT, 4, 4, 4, { VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT, VK_FORMAT_R32_SFLOAT } },
345 
346  /* Two-plane 420 YUV at 8, 10, 12 and 16 bits */
347  { VK_FORMAT_G8_B8R8_2PLANE_420_UNORM, AV_PIX_FMT_NV12, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
348  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P010, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
349  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_420_UNORM_3PACK16, AV_PIX_FMT_P012, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
350  { VK_FORMAT_G16_B16R16_2PLANE_420_UNORM, AV_PIX_FMT_P016, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
351 
352  /* Two-plane 422 YUV at 8, 10 and 16 bits */
353  { VK_FORMAT_G8_B8R8_2PLANE_422_UNORM, AV_PIX_FMT_NV16, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
354  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P210, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
355  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_422_UNORM_3PACK16, AV_PIX_FMT_P212, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
356  { VK_FORMAT_G16_B16R16_2PLANE_422_UNORM, AV_PIX_FMT_P216, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
357 
358  /* Two-plane 444 YUV at 8, 10 and 16 bits */
359  { VK_FORMAT_G8_B8R8_2PLANE_444_UNORM, AV_PIX_FMT_NV24, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8G8_UNORM } },
360  { VK_FORMAT_G10X6_B10X6R10X6_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P410, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
361  { VK_FORMAT_G12X4_B12X4R12X4_2PLANE_444_UNORM_3PACK16, AV_PIX_FMT_P412, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
362  { VK_FORMAT_G16_B16R16_2PLANE_444_UNORM, AV_PIX_FMT_P416, ASPECT_2PLANE, 2, 1, 2, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16G16_UNORM } },
363 
364  /* Three-plane 420, 422, 444 at 8, 10, 12 and 16 bits */
365  { VK_FORMAT_G8_B8_R8_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
366  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
367  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
368  { VK_FORMAT_G16_B16_R16_3PLANE_420_UNORM, AV_PIX_FMT_YUV420P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
369  { VK_FORMAT_G8_B8_R8_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
370  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
371  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
372  { VK_FORMAT_G16_B16_R16_3PLANE_422_UNORM, AV_PIX_FMT_YUV422P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
373  { VK_FORMAT_G8_B8_R8_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM, VK_FORMAT_R8_UNORM } },
374  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P10, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
375  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P12, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
376  { VK_FORMAT_G16_B16_R16_3PLANE_444_UNORM, AV_PIX_FMT_YUV444P16, ASPECT_3PLANE, 3, 1, 3, { VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM, VK_FORMAT_R16_UNORM } },
377 
378  /* Single plane 422 at 8, 10, 12 and 16 bits */
379  { VK_FORMAT_G8B8G8R8_422_UNORM, AV_PIX_FMT_YUYV422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
380  { VK_FORMAT_B8G8R8G8_422_UNORM, AV_PIX_FMT_UYVY422, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R8G8B8A8_UNORM } },
381  { VK_FORMAT_G10X6B10X6G10X6R10X6_422_UNORM_4PACK16, AV_PIX_FMT_Y210, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
382  { VK_FORMAT_G12X4B12X4G12X4R12X4_422_UNORM_4PACK16, AV_PIX_FMT_Y212, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
383  { VK_FORMAT_G16B16G16R16_422_UNORM, AV_PIX_FMT_Y216, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
384 
385  /* Single plane 444 at 8, 10, 12 and 16 bits */
386  { VK_FORMAT_B8G8R8A8_UNORM, AV_PIX_FMT_UYVA, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_B8G8R8A8_UNORM } },
387  { VK_FORMAT_A2R10G10B10_UNORM_PACK32, AV_PIX_FMT_XV30, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
388  { VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16, AV_PIX_FMT_XV36, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
389  { VK_FORMAT_R16G16B16A16_UNORM, AV_PIX_FMT_XV48, VK_IMAGE_ASPECT_COLOR_BIT, 1, 1, 1, { VK_FORMAT_R16G16B16A16_UNORM } },
390 };
391 static const int nb_vk_formats_list = FF_ARRAY_ELEMS(vk_formats_list);
392 
393 const VkFormat *av_vkfmt_from_pixfmt(enum AVPixelFormat p)
394 {
395  for (int i = 0; i < nb_vk_formats_list; i++)
396  if (vk_formats_list[i].pixfmt == p)
397  return vk_formats_list[i].fallback;
398  return NULL;
399 }
400 
401 static const struct FFVkFormatEntry *vk_find_format_entry(enum AVPixelFormat p)
402 {
403  for (int i = 0; i < nb_vk_formats_list; i++)
404  if (vk_formats_list[i].pixfmt == p)
405  return &vk_formats_list[i];
406  return NULL;
407 }
408 
409 /* Malitia pura, Khronos */
410 #define FN_MAP_TO(dst_t, dst_name, src_t, src_name) \
411  static av_unused dst_t map_ ##src_name## _to_ ##dst_name(src_t src) \
412  { \
413  dst_t dst = 0x0; \
414  MAP_TO(VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT, \
415  VK_IMAGE_USAGE_SAMPLED_BIT); \
416  MAP_TO(VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT, \
417  VK_IMAGE_USAGE_TRANSFER_SRC_BIT); \
418  MAP_TO(VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT, \
419  VK_IMAGE_USAGE_TRANSFER_DST_BIT); \
420  MAP_TO(VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT, \
421  VK_IMAGE_USAGE_STORAGE_BIT); \
422  MAP_TO(VK_FORMAT_FEATURE_2_COLOR_ATTACHMENT_BIT, \
423  VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT); \
424  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_DECODE_OUTPUT_BIT_KHR, \
425  VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR); \
426  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_DECODE_DPB_BIT_KHR, \
427  VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR); \
428  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_ENCODE_DPB_BIT_KHR, \
429  VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR); \
430  MAP_TO(VK_FORMAT_FEATURE_2_VIDEO_ENCODE_INPUT_BIT_KHR, \
431  VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR); \
432  return dst; \
433  }
434 
435 #define MAP_TO(flag1, flag2) if (src & flag2) dst |= flag1;
436 FN_MAP_TO(VkFormatFeatureFlagBits2, feats, VkImageUsageFlags, usage)
437 #undef MAP_TO
438 #define MAP_TO(flag1, flag2) if (src & flag1) dst |= flag2;
439 FN_MAP_TO(VkImageUsageFlags, usage, VkFormatFeatureFlagBits2, feats)
440 #undef MAP_TO
441 #undef FN_MAP_TO
442 
443 static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p,
444  VkImageTiling tiling,
445  VkFormat fmts[AV_NUM_DATA_POINTERS], /* Output format list */
446  int *nb_images, /* Output number of images */
447  VkImageAspectFlags *aspect, /* Output aspect */
448  VkImageUsageFlags *supported_usage, /* Output supported usage */
449  int disable_multiplane, int need_storage)
450 {
451  VulkanDevicePriv *priv = dev_ctx->hwctx;
452  AVVulkanDeviceContext *hwctx = &priv->p;
453  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
454 
455  const VkFormatFeatureFlagBits2 basic_flags = VK_FORMAT_FEATURE_2_SAMPLED_IMAGE_BIT |
456  VK_FORMAT_FEATURE_2_TRANSFER_SRC_BIT |
457  VK_FORMAT_FEATURE_2_TRANSFER_DST_BIT;
458 
459  for (int i = 0; i < nb_vk_formats_list; i++) {
460  if (vk_formats_list[i].pixfmt == p) {
461  VkFormatProperties3 fprops = {
462  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_3,
463  };
464  VkFormatProperties2 prop = {
465  .sType = VK_STRUCTURE_TYPE_FORMAT_PROPERTIES_2,
466  .pNext = &fprops,
467  };
468  VkFormatFeatureFlagBits2 feats_primary, feats_secondary;
469  int basics_primary = 0, basics_secondary = 0;
470  int storage_primary = 0, storage_secondary = 0;
471 
472  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
473  vk_formats_list[i].vkf,
474  &prop);
475 
476  feats_primary = tiling == VK_IMAGE_TILING_LINEAR ?
477  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
478  basics_primary = (feats_primary & basic_flags) == basic_flags;
479  storage_primary = !!(feats_primary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
480 
481  if (vk_formats_list[i].vkf != vk_formats_list[i].fallback[0]) {
482  vk->GetPhysicalDeviceFormatProperties2(hwctx->phys_dev,
483  vk_formats_list[i].fallback[0],
484  &prop);
485  feats_secondary = tiling == VK_IMAGE_TILING_LINEAR ?
486  fprops.linearTilingFeatures : fprops.optimalTilingFeatures;
487  basics_secondary = (feats_secondary & basic_flags) == basic_flags;
488  storage_secondary = !!(feats_secondary & VK_FORMAT_FEATURE_2_STORAGE_IMAGE_BIT);
489  } else {
490  basics_secondary = basics_primary;
491  storage_secondary = storage_primary;
492  }
493 
494  if (basics_primary &&
495  !(disable_multiplane && vk_formats_list[i].vk_planes > 1) &&
496  (!need_storage || (need_storage && (storage_primary | storage_secondary)))) {
497  if (fmts) {
498  if (vk_formats_list[i].nb_images > 1) {
499  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
500  fmts[j] = vk_formats_list[i].fallback[j];
501  } else {
502  fmts[0] = vk_formats_list[i].vkf;
503  }
504  }
505  if (nb_images)
506  *nb_images = 1;
507  if (aspect)
508  *aspect = vk_formats_list[i].aspect;
509  if (supported_usage)
510  *supported_usage = map_feats_to_usage(feats_primary) |
511  ((need_storage && (storage_primary | storage_secondary)) ?
512  VK_IMAGE_USAGE_STORAGE_BIT : 0);
513  return 0;
514  } else if (basics_secondary &&
515  (!need_storage || (need_storage && storage_secondary))) {
516  if (fmts) {
517  for (int j = 0; j < vk_formats_list[i].nb_images_fallback; j++)
518  fmts[j] = vk_formats_list[i].fallback[j];
519  }
520  if (nb_images)
521  *nb_images = vk_formats_list[i].nb_images_fallback;
522  if (aspect)
523  *aspect = vk_formats_list[i].aspect;
524  if (supported_usage)
525  *supported_usage = map_feats_to_usage(feats_secondary);
526  return 0;
527  } else {
528  return AVERROR(ENOTSUP);
529  }
530  }
531  }
532 
533  return AVERROR(EINVAL);
534 }
535 
536 static int load_libvulkan(AVHWDeviceContext *ctx)
537 {
538  VulkanDevicePriv *p = ctx->hwctx;
539  AVVulkanDeviceContext *hwctx = &p->p;
540 
541  static const char *lib_names[] = {
542 #if defined(_WIN32)
543  "vulkan-1.dll",
544 #elif defined(__APPLE__)
545  "libvulkan.dylib",
546  "libvulkan.1.dylib",
547  "libMoltenVK.dylib",
548 #else
549  "libvulkan.so.1",
550  "libvulkan.so",
551 #endif
552  };
553 
554  for (int i = 0; i < FF_ARRAY_ELEMS(lib_names); i++) {
555  p->libvulkan = dlopen(lib_names[i], RTLD_NOW | RTLD_LOCAL);
556  if (p->libvulkan)
557  break;
558  }
559 
560  if (!p->libvulkan) {
561  av_log(ctx, AV_LOG_ERROR, "Unable to open the libvulkan library!\n");
562  return AVERROR_UNKNOWN;
563  }
564 
565  hwctx->get_proc_addr = (PFN_vkGetInstanceProcAddr)dlsym(p->libvulkan, "vkGetInstanceProcAddr");
566 
567  return 0;
568 }
569 
570 typedef struct VulkanOptExtension {
571  const char *name;
572  FFVulkanExtensions flag;
573 } VulkanOptExtension;
574 
575 static const VulkanOptExtension optional_instance_exts[] = {
576  { VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME, FF_VK_EXT_NO_FLAG },
577 };
578 
579 static const VulkanOptExtension optional_device_exts[] = {
580  /* Misc or required by other extensions */
581  { VK_KHR_PORTABILITY_SUBSET_EXTENSION_NAME, FF_VK_EXT_PORTABILITY_SUBSET },
582  { VK_KHR_PUSH_DESCRIPTOR_EXTENSION_NAME, FF_VK_EXT_PUSH_DESCRIPTOR },
583  { VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME, FF_VK_EXT_DESCRIPTOR_BUFFER },
584  { VK_EXT_PHYSICAL_DEVICE_DRM_EXTENSION_NAME, FF_VK_EXT_DEVICE_DRM },
585  { VK_EXT_SHADER_ATOMIC_FLOAT_EXTENSION_NAME, FF_VK_EXT_ATOMIC_FLOAT },
586  { VK_KHR_COOPERATIVE_MATRIX_EXTENSION_NAME, FF_VK_EXT_COOP_MATRIX },
587  { VK_NV_OPTICAL_FLOW_EXTENSION_NAME, FF_VK_EXT_OPTICAL_FLOW },
588  { VK_EXT_SHADER_OBJECT_EXTENSION_NAME, FF_VK_EXT_SHADER_OBJECT },
589  { VK_KHR_VIDEO_MAINTENANCE_1_EXTENSION_NAME, FF_VK_EXT_VIDEO_MAINTENANCE_1 },
590 
591  /* Imports/exports */
592  { VK_KHR_EXTERNAL_MEMORY_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_MEMORY },
593  { VK_EXT_EXTERNAL_MEMORY_DMA_BUF_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_DMABUF_MEMORY },
594  { VK_EXT_IMAGE_DRM_FORMAT_MODIFIER_EXTENSION_NAME, FF_VK_EXT_DRM_MODIFIER_FLAGS },
595  { VK_KHR_EXTERNAL_SEMAPHORE_FD_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_FD_SEM },
596  { VK_EXT_EXTERNAL_MEMORY_HOST_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_HOST_MEMORY },
597 #ifdef _WIN32
598  { VK_KHR_EXTERNAL_MEMORY_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_MEMORY },
599  { VK_KHR_EXTERNAL_SEMAPHORE_WIN32_EXTENSION_NAME, FF_VK_EXT_EXTERNAL_WIN32_SEM },
600 #endif
601 
602  /* Video encoding/decoding */
603  { VK_KHR_VIDEO_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_QUEUE },
604  { VK_KHR_VIDEO_ENCODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_QUEUE },
605  { VK_KHR_VIDEO_DECODE_QUEUE_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_QUEUE },
606  { VK_KHR_VIDEO_ENCODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H264 },
607  { VK_KHR_VIDEO_DECODE_H264_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H264 },
608  { VK_KHR_VIDEO_ENCODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_ENCODE_H265 },
609  { VK_KHR_VIDEO_DECODE_H265_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_H265 },
610  { VK_KHR_VIDEO_DECODE_AV1_EXTENSION_NAME, FF_VK_EXT_VIDEO_DECODE_AV1 },
611 };
612 
613 static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity,
614  VkDebugUtilsMessageTypeFlagsEXT messageType,
615  const VkDebugUtilsMessengerCallbackDataEXT *data,
616  void *priv)
617 {
618  int l;
619  AVHWDeviceContext *ctx = priv;
620 
621  /* Ignore false positives */
622  switch (data->messageIdNumber) {
623  case 0x086974c1: /* BestPractices-vkCreateCommandPool-command-buffer-reset */
624  case 0xfd92477a: /* BestPractices-vkAllocateMemory-small-allocation */
625  case 0x618ab1e7: /* VUID-VkImageViewCreateInfo-usage-02275 */
626  case 0x30f4ac70: /* VUID-VkImageCreateInfo-pNext-06811 */
627  return VK_FALSE;
628  default:
629  break;
630  }
631 
632  switch (severity) {
633  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT: l = AV_LOG_VERBOSE; break;
634  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT: l = AV_LOG_INFO; break;
635  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT: l = AV_LOG_WARNING; break;
636  case VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT: l = AV_LOG_ERROR; break;
637  default: l = AV_LOG_DEBUG; break;
638  }
639 
640  av_log(ctx, l, "%s\n", data->pMessage);
641  for (int i = 0; i < data->cmdBufLabelCount; i++)
642  av_log(ctx, l, "\t%i: %s\n", i, data->pCmdBufLabels[i].pLabelName);
643 
644  return VK_FALSE;
645 }
646 
647 #define ADD_VAL_TO_LIST(list, count, val) \
648  do { \
649  list = av_realloc_array(list, sizeof(*list), ++count); \
650  if (!list) { \
651  err = AVERROR(ENOMEM); \
652  goto fail; \
653  } \
654  list[count - 1] = av_strdup(val); \
655  if (!list[count - 1]) { \
656  err = AVERROR(ENOMEM); \
657  goto fail; \
658  } \
659  } while(0)
660 
661 #define RELEASE_PROPS(props, count) \
662  if (props) { \
663  for (int i = 0; i < count; i++) \
664  av_free((void *)((props)[i])); \
665  av_free((void *)props); \
666  }
667 
668 enum FFVulkanDebugMode {
669  FF_VULKAN_DEBUG_NONE = 0,
670  /* Standard GPU-assisted validation */
671  FF_VULKAN_DEBUG_VALIDATE = 1,
672  /* Passes printfs in shaders to the debug callback */
673  FF_VULKAN_DEBUG_PRINTF = 2,
674  /* Enables extra printouts */
675  FF_VULKAN_DEBUG_PRACTICES = 3,
676  /* Disables validation but keeps shader debug info and optimizations */
677  FF_VULKAN_DEBUG_PROFILE = 4,
678 
679  FF_VULKAN_DEBUG_NB,
680 };
681 
682 static int check_extensions(AVHWDeviceContext *ctx, int dev, AVDictionary *opts,
683  const char * const **dst, uint32_t *num,
684  enum FFVulkanDebugMode debug_mode)
685 {
686  const char *tstr;
687  const char **extension_names = NULL;
688  VulkanDevicePriv *p = ctx->hwctx;
689  AVVulkanDeviceContext *hwctx = &p->p;
690  FFVulkanFunctions *vk = &p->vkctx.vkfn;
691  int err = 0, found, extensions_found = 0;
692 
693  const char *mod;
694  int optional_exts_num;
695  uint32_t sup_ext_count;
696  char *user_exts_str = NULL;
697  AVDictionaryEntry *user_exts;
698  VkExtensionProperties *sup_ext;
699  const VulkanOptExtension *optional_exts;
700 
701  if (!dev) {
702  mod = "instance";
703  optional_exts = optional_instance_exts;
704  optional_exts_num = FF_ARRAY_ELEMS(optional_instance_exts);
705  user_exts = av_dict_get(opts, "instance_extensions", NULL, 0);
706  if (user_exts) {
707  user_exts_str = av_strdup(user_exts->value);
708  if (!user_exts_str) {
709  err = AVERROR(ENOMEM);
710  goto fail;
711  }
712  }
713  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, NULL);
714  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
715  if (!sup_ext)
716  return AVERROR(ENOMEM);
717  vk->EnumerateInstanceExtensionProperties(NULL, &sup_ext_count, sup_ext);
718  } else {
719  mod = "device";
720  optional_exts = optional_device_exts;
721  optional_exts_num = FF_ARRAY_ELEMS(optional_device_exts);
722  user_exts = av_dict_get(opts, "device_extensions", NULL, 0);
723  if (user_exts) {
724  user_exts_str = av_strdup(user_exts->value);
725  if (!user_exts_str) {
726  err = AVERROR(ENOMEM);
727  goto fail;
728  }
729  }
730  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
731  &sup_ext_count, NULL);
732  sup_ext = av_malloc_array(sup_ext_count, sizeof(VkExtensionProperties));
733  if (!sup_ext)
734  return AVERROR(ENOMEM);
735  vk->EnumerateDeviceExtensionProperties(hwctx->phys_dev, NULL,
736  &sup_ext_count, sup_ext);
737  }
738 
739  for (int i = 0; i < optional_exts_num; i++) {
740  tstr = optional_exts[i].name;
741  found = 0;
742 
743  if (dev &&
744  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
745  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
746  (debug_mode == FF_VULKAN_DEBUG_PRACTICES)) &&
747  !strcmp(tstr, VK_EXT_DESCRIPTOR_BUFFER_EXTENSION_NAME)) {
748  continue;
749  }
750 
751  for (int j = 0; j < sup_ext_count; j++) {
752  if (!strcmp(tstr, sup_ext[j].extensionName)) {
753  found = 1;
754  break;
755  }
756  }
757  if (!found)
758  continue;
759 
760  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
761  p->vkctx.extensions |= optional_exts[i].flag;
762  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
763  }
764 
765  if (!dev &&
766  ((debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
767  (debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
768  (debug_mode == FF_VULKAN_DEBUG_PRACTICES))) {
769  tstr = VK_EXT_DEBUG_UTILS_EXTENSION_NAME;
770  found = 0;
771  for (int j = 0; j < sup_ext_count; j++) {
772  if (!strcmp(tstr, sup_ext[j].extensionName)) {
773  found = 1;
774  break;
775  }
776  }
777  if (found) {
778  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
779  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
780  } else {
781  av_log(ctx, AV_LOG_ERROR, "Debug extension \"%s\" not found!\n",
782  tstr);
783  err = AVERROR(EINVAL);
784  goto fail;
785  }
786  }
787 
788 #ifdef VK_KHR_shader_relaxed_extended_instruction
789  if (((debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
790  (debug_mode == FF_VULKAN_DEBUG_PROFILE)) && dev) {
791  tstr = VK_KHR_SHADER_RELAXED_EXTENDED_INSTRUCTION_EXTENSION_NAME;
792  found = 0;
793  for (int j = 0; j < sup_ext_count; j++) {
794  if (!strcmp(tstr, sup_ext[j].extensionName)) {
795  found = 1;
796  break;
797  }
798  }
799  if (found) {
800  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension %s\n", mod, tstr);
801  ADD_VAL_TO_LIST(extension_names, extensions_found, tstr);
802  } else {
803  av_log(ctx, AV_LOG_ERROR, "Debug_printf/profile enabled, but extension \"%s\" not found!\n",
804  tstr);
805  err = AVERROR(EINVAL);
806  goto fail;
807  }
808  }
809 #endif
810 
811  if (user_exts_str) {
812  char *save, *token = av_strtok(user_exts_str, "+", &save);
813  while (token) {
814  found = 0;
815  for (int j = 0; j < sup_ext_count; j++) {
816  if (!strcmp(token, sup_ext[j].extensionName)) {
817  found = 1;
818  break;
819  }
820  }
821  if (found) {
822  av_log(ctx, AV_LOG_VERBOSE, "Using %s extension \"%s\"\n", mod, token);
823  ADD_VAL_TO_LIST(extension_names, extensions_found, token);
824  } else {
825  av_log(ctx, AV_LOG_WARNING, "%s extension \"%s\" not found, excluding.\n",
826  mod, token);
827  }
828  token = av_strtok(NULL, "+", &save);
829  }
830  }
831 
832  *dst = extension_names;
833  *num = extensions_found;
834 
835  av_free(user_exts_str);
836  av_free(sup_ext);
837  return 0;
838 
839 fail:
840  RELEASE_PROPS(extension_names, extensions_found);
841  av_free(user_exts_str);
842  av_free(sup_ext);
843  return err;
844 }
845 
846 static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts,
847  const char * const **dst, uint32_t *num,
848  enum FFVulkanDebugMode *debug_mode)
849 {
850  int err = 0;
851  VulkanDevicePriv *priv = ctx->hwctx;
852  FFVulkanFunctions *vk = &priv->vkctx.vkfn;
853 
854  static const char layer_standard_validation[] = { "VK_LAYER_KHRONOS_validation" };
855  int layer_standard_validation_found = 0;
856 
857  uint32_t sup_layer_count;
858  VkLayerProperties *sup_layers;
859 
860  AVDictionaryEntry *user_layers = av_dict_get(opts, "layers", NULL, 0);
861  char *user_layers_str = NULL;
862  char *save, *token;
863 
864  const char **enabled_layers = NULL;
865  uint32_t enabled_layers_count = 0;
866 
867  AVDictionaryEntry *debug_opt = av_dict_get(opts, "debug", NULL, 0);
868  enum FFVulkanDebugMode mode;
869 
870  *debug_mode = mode = FF_VULKAN_DEBUG_NONE;
871 
872  /* Get a list of all layers */
873  vk->EnumerateInstanceLayerProperties(&sup_layer_count, NULL);
874  sup_layers = av_malloc_array(sup_layer_count, sizeof(VkLayerProperties));
875  if (!sup_layers)
876  return AVERROR(ENOMEM);
877  vk->EnumerateInstanceLayerProperties(&sup_layer_count, sup_layers);
878 
879  av_log(ctx, AV_LOG_VERBOSE, "Supported layers:\n");
880  for (int i = 0; i < sup_layer_count; i++)
881  av_log(ctx, AV_LOG_VERBOSE, "\t%s\n", sup_layers[i].layerName);
882 
883  /* If no user layers or debug layers are given, return */
884  if (!debug_opt && !user_layers)
885  goto end;
886 
887  /* Check for any properly supported validation layer */
888  if (debug_opt) {
889  if (!strcmp(debug_opt->value, "profile")) {
890  mode = FF_VULKAN_DEBUG_PROFILE;
891  } else if (!strcmp(debug_opt->value, "printf")) {
892  mode = FF_VULKAN_DEBUG_PRINTF;
893  } else if (!strcmp(debug_opt->value, "validate")) {
894  mode = FF_VULKAN_DEBUG_VALIDATE;
895  } else if (!strcmp(debug_opt->value, "practices")) {
896  mode = FF_VULKAN_DEBUG_PRACTICES;
897  } else {
898  char *end_ptr = NULL;
899  int idx = strtol(debug_opt->value, &end_ptr, 10);
900  if (end_ptr == debug_opt->value || end_ptr[0] != '\0' ||
901  idx < 0 || idx >= FF_VULKAN_DEBUG_NB) {
902  av_log(ctx, AV_LOG_ERROR, "Invalid debugging mode \"%s\"\n",
903  debug_opt->value);
904  err = AVERROR(EINVAL);
905  goto end;
906  }
907  mode = idx;
908  }
909  }
910 
911  /* If mode is VALIDATE or PRINTF, try to find the standard validation layer extension */
912  if ((mode == FF_VULKAN_DEBUG_VALIDATE) ||
913  (mode == FF_VULKAN_DEBUG_PRINTF) ||
914  (mode == FF_VULKAN_DEBUG_PRACTICES)) {
915  for (int i = 0; i < sup_layer_count; i++) {
916  if (!strcmp(layer_standard_validation, sup_layers[i].layerName)) {
917  av_log(ctx, AV_LOG_VERBOSE, "Standard validation layer %s is enabled\n",
918  layer_standard_validation);
919  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, layer_standard_validation);
920  *debug_mode = mode;
921  layer_standard_validation_found = 1;
922  break;
923  }
924  }
925  if (!layer_standard_validation_found) {
926  av_log(ctx, AV_LOG_ERROR,
927  "Validation Layer \"%s\" not supported\n", layer_standard_validation);
928  err = AVERROR(ENOTSUP);
929  goto end;
930  }
931  } else if (mode == FF_VULKAN_DEBUG_PROFILE) {
932  *debug_mode = mode;
933  }
934 
935  /* Process any custom layers enabled */
936  if (user_layers) {
937  int found;
938 
939  user_layers_str = av_strdup(user_layers->value);
940  if (!user_layers_str) {
941  err = AVERROR(ENOMEM);
942  goto fail;
943  }
944 
945  token = av_strtok(user_layers_str, "+", &save);
946  while (token) {
947  found = 0;
948 
949  /* If debug=1/2 was specified as an option, skip this layer */
950  if (!strcmp(layer_standard_validation, token) && layer_standard_validation_found) {
951  token = av_strtok(NULL, "+", &save);
952  break;
953  }
954 
955  /* Try to find the layer in the list of supported layers */
956  for (int j = 0; j < sup_layer_count; j++) {
957  if (!strcmp(token, sup_layers[j].layerName)) {
958  found = 1;
959  break;
960  }
961  }
962 
963  if (found) {
964  av_log(ctx, AV_LOG_VERBOSE, "Using layer: %s\n", token);
965  ADD_VAL_TO_LIST(enabled_layers, enabled_layers_count, token);
966 
967  /* If debug was not set as an option, force it */
968  if (!strcmp(layer_standard_validation, token))
969  *debug_mode = FF_VULKAN_DEBUG_VALIDATE;
970  } else {
971  av_log(ctx, AV_LOG_ERROR,
972  "Layer \"%s\" not supported\n", token);
973  err = AVERROR(EINVAL);
974  goto end;
975  }
976 
977  token = av_strtok(NULL, "+", &save);
978  }
979  }
980 
981 fail:
982 end:
983  av_free(sup_layers);
984  av_free(user_layers_str);
985 
986  if (err < 0) {
987  RELEASE_PROPS(enabled_layers, enabled_layers_count);
988  } else {
989  *dst = enabled_layers;
990  *num = enabled_layers_count;
991  }
992 
993  return err;
994 }
995 
996 /* Creates a VkInstance */
997 static int create_instance(AVHWDeviceContext *ctx, AVDictionary *opts,
998  enum FFVulkanDebugMode *debug_mode)
999 {
1000  int err = 0;
1001  VkResult ret;
1002  VulkanDevicePriv *p = ctx->hwctx;
1003  AVVulkanDeviceContext *hwctx = &p->p;
1004  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1005  VkApplicationInfo application_info = {
1006  .sType = VK_STRUCTURE_TYPE_APPLICATION_INFO,
1007  .pApplicationName = "ffmpeg",
1008  .applicationVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1009  LIBAVUTIL_VERSION_MINOR,
1010  LIBAVUTIL_VERSION_MICRO),
1011  .pEngineName = "libavutil",
1012  .apiVersion = VK_API_VERSION_1_3,
1013  .engineVersion = VK_MAKE_VERSION(LIBAVUTIL_VERSION_MAJOR,
1014  LIBAVUTIL_VERSION_MINOR,
1015  LIBAVUTIL_VERSION_MICRO),
1016  };
1017  VkValidationFeaturesEXT validation_features = {
1018  .sType = VK_STRUCTURE_TYPE_VALIDATION_FEATURES_EXT,
1019  };
1020  VkInstanceCreateInfo inst_props = {
1021  .sType = VK_STRUCTURE_TYPE_INSTANCE_CREATE_INFO,
1022  .pApplicationInfo = &application_info,
1023  };
1024 
1025  if (!hwctx->get_proc_addr) {
1026  err = load_libvulkan(ctx);
1027  if (err < 0)
1028  return err;
1029  }
1030 
1031  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 0, 0);
1032  if (err < 0) {
1033  av_log(ctx, AV_LOG_ERROR, "Unable to load instance enumeration functions!\n");
1034  return err;
1035  }
1036 
1037  err = check_layers(ctx, opts, &inst_props.ppEnabledLayerNames,
1038  &inst_props.enabledLayerCount, debug_mode);
1039  if (err)
1040  goto fail;
1041 
1042  /* Check for present/missing extensions */
1043  err = check_extensions(ctx, 0, opts, &inst_props.ppEnabledExtensionNames,
1044  &inst_props.enabledExtensionCount, *debug_mode);
1045  hwctx->enabled_inst_extensions = inst_props.ppEnabledExtensionNames;
1046  hwctx->nb_enabled_inst_extensions = inst_props.enabledExtensionCount;
1047  if (err < 0)
1048  goto fail;
1049 
1050  /* Enable debug features if needed */
1051  if (*debug_mode == FF_VULKAN_DEBUG_VALIDATE) {
1052  static const VkValidationFeatureEnableEXT feat_list_validate[] = {
1053  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1054  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1055  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_EXT,
1056  };
1057  validation_features.pEnabledValidationFeatures = feat_list_validate;
1058  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_validate);
1059  inst_props.pNext = &validation_features;
1060  } else if (*debug_mode == FF_VULKAN_DEBUG_PRINTF) {
1061  static const VkValidationFeatureEnableEXT feat_list_debug[] = {
1062  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1063  VK_VALIDATION_FEATURE_ENABLE_GPU_ASSISTED_RESERVE_BINDING_SLOT_EXT,
1064  VK_VALIDATION_FEATURE_ENABLE_DEBUG_PRINTF_EXT,
1065  };
1066  validation_features.pEnabledValidationFeatures = feat_list_debug;
1067  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_debug);
1068  inst_props.pNext = &validation_features;
1069  } else if (*debug_mode == FF_VULKAN_DEBUG_PRACTICES) {
1070  static const VkValidationFeatureEnableEXT feat_list_practices[] = {
1071  VK_VALIDATION_FEATURE_ENABLE_SYNCHRONIZATION_VALIDATION_EXT,
1072  VK_VALIDATION_FEATURE_ENABLE_BEST_PRACTICES_EXT,
1073  };
1074  validation_features.pEnabledValidationFeatures = feat_list_practices;
1075  validation_features.enabledValidationFeatureCount = FF_ARRAY_ELEMS(feat_list_practices);
1076  inst_props.pNext = &validation_features;
1077  }
1078 
1079 #ifdef __APPLE__
1080  for (int i = 0; i < inst_props.enabledExtensionCount; i++) {
1081  if (!strcmp(VK_KHR_PORTABILITY_ENUMERATION_EXTENSION_NAME,
1082  inst_props.ppEnabledExtensionNames[i])) {
1083  inst_props.flags |= VK_INSTANCE_CREATE_ENUMERATE_PORTABILITY_BIT_KHR;
1084  break;
1085  }
1086  }
1087 #endif
1088 
1089  /* Try to create the instance */
1090  ret = vk->CreateInstance(&inst_props, hwctx->alloc, &hwctx->inst);
1091 
1092  /* Check for errors */
1093  if (ret != VK_SUCCESS) {
1094  av_log(ctx, AV_LOG_ERROR, "Instance creation failure: %s\n",
1095  ff_vk_ret2str(ret));
1096  err = AVERROR_EXTERNAL;
1097  goto fail;
1098  }
1099 
1100  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 0);
1101  if (err < 0) {
1102  av_log(ctx, AV_LOG_ERROR, "Unable to load instance functions!\n");
1103  goto fail;
1104  }
1105 
1106  /* Setup debugging callback if needed */
1107  if ((*debug_mode == FF_VULKAN_DEBUG_VALIDATE) ||
1108  (*debug_mode == FF_VULKAN_DEBUG_PRINTF) ||
1109  (*debug_mode == FF_VULKAN_DEBUG_PRACTICES)) {
1110  VkDebugUtilsMessengerCreateInfoEXT dbg = {
1111  .sType = VK_STRUCTURE_TYPE_DEBUG_UTILS_MESSENGER_CREATE_INFO_EXT,
1112  .messageSeverity = VK_DEBUG_UTILS_MESSAGE_SEVERITY_VERBOSE_BIT_EXT |
1113  VK_DEBUG_UTILS_MESSAGE_SEVERITY_INFO_BIT_EXT |
1114  VK_DEBUG_UTILS_MESSAGE_SEVERITY_WARNING_BIT_EXT |
1115  VK_DEBUG_UTILS_MESSAGE_SEVERITY_ERROR_BIT_EXT,
1116  .messageType = VK_DEBUG_UTILS_MESSAGE_TYPE_GENERAL_BIT_EXT |
1117  VK_DEBUG_UTILS_MESSAGE_TYPE_VALIDATION_BIT_EXT |
1118  VK_DEBUG_UTILS_MESSAGE_TYPE_PERFORMANCE_BIT_EXT,
1119  .pfnUserCallback = vk_dbg_callback,
1120  .pUserData = ctx,
1121  };
1122 
1123  vk->CreateDebugUtilsMessengerEXT(hwctx->inst, &dbg,
1124  hwctx->alloc, &p->debug_ctx);
1125  }
1126 
1127  err = 0;
1128 
1129 fail:
1130  RELEASE_PROPS(inst_props.ppEnabledLayerNames, inst_props.enabledLayerCount);
1131  return err;
1132 }
1133 
1134 typedef struct VulkanDeviceSelection {
1135  uint8_t uuid[VK_UUID_SIZE]; /* Will use this first unless !has_uuid */
1136  int has_uuid;
1137  uint32_t drm_major; /* Will use this second unless !has_drm */
1138  uint32_t drm_minor; /* Will use this second unless !has_drm */
1139  uint32_t has_drm; /* has drm node info */
1140  const char *name; /* Will use this third unless NULL */
1141  uint32_t pci_device; /* Will use this fourth unless 0x0 */
1142  uint32_t vendor_id; /* Last resort to find something deterministic */
1143  int index; /* Finally fall back to index */
1144 } VulkanDeviceSelection;
1145 
1146 static const char *vk_dev_type(enum VkPhysicalDeviceType type)
1147 {
1148  switch (type) {
1149  case VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU: return "integrated";
1150  case VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU: return "discrete";
1151  case VK_PHYSICAL_DEVICE_TYPE_VIRTUAL_GPU: return "virtual";
1152  case VK_PHYSICAL_DEVICE_TYPE_CPU: return "software";
1153  default: return "unknown";
1154  }
1155 }
1156 
1157 /* Finds a device */
1158 static int find_device(AVHWDeviceContext *ctx, VulkanDeviceSelection *select)
1159 {
1160  int err = 0, choice = -1;
1161  uint32_t num;
1162  VkResult ret;
1163  VulkanDevicePriv *p = ctx->hwctx;
1164  AVVulkanDeviceContext *hwctx = &p->p;
1165  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1166  VkPhysicalDevice *devices = NULL;
1167  VkPhysicalDeviceIDProperties *idp = NULL;
1168  VkPhysicalDeviceProperties2 *prop = NULL;
1169  VkPhysicalDeviceDrmPropertiesEXT *drm_prop = NULL;
1170 
1171  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, NULL);
1172  if (ret != VK_SUCCESS || !num) {
1173  av_log(ctx, AV_LOG_ERROR, "No devices found: %s!\n", ff_vk_ret2str(ret));
1174  return AVERROR(ENODEV);
1175  }
1176 
1177  devices = av_malloc_array(num, sizeof(VkPhysicalDevice));
1178  if (!devices)
1179  return AVERROR(ENOMEM);
1180 
1181  ret = vk->EnumeratePhysicalDevices(hwctx->inst, &num, devices);
1182  if (ret != VK_SUCCESS) {
1183  av_log(ctx, AV_LOG_ERROR, "Failed enumerating devices: %s\n",
1184  ff_vk_ret2str(ret));
1185  err = AVERROR(ENODEV);
1186  goto end;
1187  }
1188 
1189  prop = av_calloc(num, sizeof(*prop));
1190  if (!prop) {
1191  err = AVERROR(ENOMEM);
1192  goto end;
1193  }
1194 
1195  idp = av_calloc(num, sizeof(*idp));
1196  if (!idp) {
1197  err = AVERROR(ENOMEM);
1198  goto end;
1199  }
1200 
1201  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1202  drm_prop = av_calloc(num, sizeof(*drm_prop));
1203  if (!drm_prop) {
1204  err = AVERROR(ENOMEM);
1205  goto end;
1206  }
1207  }
1208 
1209  av_log(ctx, AV_LOG_VERBOSE, "GPU listing:\n");
1210  for (int i = 0; i < num; i++) {
1211  if (p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) {
1212  drm_prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRM_PROPERTIES_EXT;
1213  idp[i].pNext = &drm_prop[i];
1214  }
1215  idp[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_ID_PROPERTIES;
1216  prop[i].sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1217  prop[i].pNext = &idp[i];
1218 
1219  vk->GetPhysicalDeviceProperties2(devices[i], &prop[i]);
1220  av_log(ctx, AV_LOG_VERBOSE, " %d: %s (%s) (0x%x)\n", i,
1221  prop[i].properties.deviceName,
1222  vk_dev_type(prop[i].properties.deviceType),
1223  prop[i].properties.deviceID);
1224  }
1225 
1226  if (select->has_uuid) {
1227  for (int i = 0; i < num; i++) {
1228  if (!strncmp(idp[i].deviceUUID, select->uuid, VK_UUID_SIZE)) {
1229  choice = i;
1230  goto end;
1231  }
1232  }
1233  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given UUID!\n");
1234  err = AVERROR(ENODEV);
1235  goto end;
1236  } else if ((p->vkctx.extensions & FF_VK_EXT_DEVICE_DRM) && select->has_drm) {
1237  for (int i = 0; i < num; i++) {
1238  if ((select->drm_major == drm_prop[i].primaryMajor &&
1239  select->drm_minor == drm_prop[i].primaryMinor) ||
1240  (select->drm_major == drm_prop[i].renderMajor &&
1241  select->drm_minor == drm_prop[i].renderMinor)) {
1242  choice = i;
1243  goto end;
1244  }
1245  }
1246  av_log(ctx, AV_LOG_ERROR, "Unable to find device by given DRM node numbers %i:%i!\n",
1247  select->drm_major, select->drm_minor);
1248  err = AVERROR(ENODEV);
1249  goto end;
1250  } else if (select->name) {
1251  av_log(ctx, AV_LOG_VERBOSE, "Requested device: %s\n", select->name);
1252  for (int i = 0; i < num; i++) {
1253  if (strstr(prop[i].properties.deviceName, select->name)) {
1254  choice = i;
1255  goto end;
1256  }
1257  }
1258  av_log(ctx, AV_LOG_ERROR, "Unable to find device \"%s\"!\n",
1259  select->name);
1260  err = AVERROR(ENODEV);
1261  goto end;
1262  } else if (select->pci_device) {
1263  av_log(ctx, AV_LOG_VERBOSE, "Requested device: 0x%x\n", select->pci_device);
1264  for (int i = 0; i < num; i++) {
1265  if (select->pci_device == prop[i].properties.deviceID) {
1266  choice = i;
1267  goto end;
1268  }
1269  }
1270  av_log(ctx, AV_LOG_ERROR, "Unable to find device with PCI ID 0x%x!\n",
1271  select->pci_device);
1272  err = AVERROR(EINVAL);
1273  goto end;
1274  } else if (select->vendor_id) {
1275  av_log(ctx, AV_LOG_VERBOSE, "Requested vendor: 0x%x\n", select->vendor_id);
1276  for (int i = 0; i < num; i++) {
1277  if (select->vendor_id == prop[i].properties.vendorID) {
1278  choice = i;
1279  goto end;
1280  }
1281  }
1282  av_log(ctx, AV_LOG_ERROR, "Unable to find device with Vendor ID 0x%x!\n",
1283  select->vendor_id);
1284  err = AVERROR(ENODEV);
1285  goto end;
1286  } else {
1287  if (select->index < num) {
1288  choice = select->index;
1289  goto end;
1290  }
1291  av_log(ctx, AV_LOG_ERROR, "Unable to find device with index %i!\n",
1292  select->index);
1293  err = AVERROR(ENODEV);
1294  goto end;
1295  }
1296 
1297 end:
1298  if (choice > -1) {
1299  av_log(ctx, AV_LOG_VERBOSE, "Device %d selected: %s (%s) (0x%x)\n",
1300  choice, prop[choice].properties.deviceName,
1301  vk_dev_type(prop[choice].properties.deviceType),
1302  prop[choice].properties.deviceID);
1303  hwctx->phys_dev = devices[choice];
1304  }
1305 
1306  av_free(devices);
1307  av_free(prop);
1308  av_free(idp);
1309  av_free(drm_prop);
1310 
1311  return err;
1312 }
1313 
1314 /* Picks the least used qf with the fewest unneeded flags, or -1 if none found */
1315 static inline int pick_queue_family(VkQueueFamilyProperties2 *qf, uint32_t num_qf,
1316  VkQueueFlagBits flags)
1317 {
1318  int index = -1;
1319  uint32_t min_score = UINT32_MAX;
1320 
1321  for (int i = 0; i < num_qf; i++) {
1322  VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1323 
1324  /* Per the spec, reporting transfer caps is optional for these 2 types */
1325  if ((flags & VK_QUEUE_TRANSFER_BIT) &&
1326  (qflags & (VK_QUEUE_GRAPHICS_BIT | VK_QUEUE_COMPUTE_BIT)))
1327  qflags |= VK_QUEUE_TRANSFER_BIT;
1328 
1329  if (qflags & flags) {
1330  uint32_t score = av_popcount(qflags) + qf[i].queueFamilyProperties.timestampValidBits;
1331  if (score < min_score) {
1332  index = i;
1333  min_score = score;
1334  }
1335  }
1336  }
1337 
1338  if (index > -1)
1339  qf[index].queueFamilyProperties.timestampValidBits++;
1340 
1341  return index;
1342 }
1343 
1344 static inline int pick_video_queue_family(VkQueueFamilyProperties2 *qf,
1345  VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf,
1346  VkVideoCodecOperationFlagBitsKHR flags)
1347 {
1348  int index = -1;
1349  uint32_t min_score = UINT32_MAX;
1350 
1351  for (int i = 0; i < num_qf; i++) {
1352  const VkQueueFlagBits qflags = qf[i].queueFamilyProperties.queueFlags;
1353  const VkQueueFlagBits vflags = qf_vid[i].videoCodecOperations;
1354 
1355  if (!(qflags & (VK_QUEUE_VIDEO_ENCODE_BIT_KHR | VK_QUEUE_VIDEO_DECODE_BIT_KHR)))
1356  continue;
1357 
1358  if (vflags & flags) {
1359  uint32_t score = av_popcount(vflags) + qf[i].queueFamilyProperties.timestampValidBits;
1360  if (score < min_score) {
1361  index = i;
1362  min_score = score;
1363  }
1364  }
1365  }
1366 
1367  if (index > -1)
1368  qf[index].queueFamilyProperties.timestampValidBits++;
1369 
1370  return index;
1371 }
1372 
1373 static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
1374 {
1375  uint32_t num;
1376  VulkanDevicePriv *p = ctx->hwctx;
1377  AVVulkanDeviceContext *hwctx = &p->p;
1378  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1379 
1380  VkQueueFamilyProperties2 *qf = NULL;
1381  VkQueueFamilyVideoPropertiesKHR *qf_vid = NULL;
1382 
1383  /* First get the number of queue families */
1384  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &num, NULL);
1385  if (!num) {
1386  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1387  return AVERROR_EXTERNAL;
1388  }
1389 
1390  /* Then allocate memory */
1391  qf = av_malloc_array(num, sizeof(VkQueueFamilyProperties2));
1392  if (!qf)
1393  return AVERROR(ENOMEM);
1394 
1395  qf_vid = av_malloc_array(num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1396  if (!qf_vid)
1397  return AVERROR(ENOMEM);
1398 
1399  for (uint32_t i = 0; i < num; i++) {
1400  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1401  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1402  };
1403  qf[i] = (VkQueueFamilyProperties2) {
1404  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1405  .pNext = &qf_vid[i],
1406  };
1407  }
1408 
1409  /* Finally retrieve the queue families */
1410  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &num, qf);
1411 
1412  av_log(ctx, AV_LOG_VERBOSE, "Queue families:\n");
1413  for (int i = 0; i < num; i++) {
1414  av_log(ctx, AV_LOG_VERBOSE, " %i:%s%s%s%s%s%s%s%s (queues: %i)\n", i,
1415  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_GRAPHICS_BIT) ? " graphics" : "",
1416  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_COMPUTE_BIT) ? " compute" : "",
1417  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_TRANSFER_BIT) ? " transfer" : "",
1418  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_ENCODE_BIT_KHR) ? " encode" : "",
1419  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_VIDEO_DECODE_BIT_KHR) ? " decode" : "",
1420  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_SPARSE_BINDING_BIT) ? " sparse" : "",
1421  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_OPTICAL_FLOW_BIT_NV) ? " optical_flow" : "",
1422  ((qf[i].queueFamilyProperties.queueFlags) & VK_QUEUE_PROTECTED_BIT) ? " protected" : "",
1423  qf[i].queueFamilyProperties.queueCount);
1424 
1425  /* We use this field to keep a score of how many times we've used that
1426  * queue family in order to make better choices. */
1427  qf[i].queueFamilyProperties.timestampValidBits = 0;
1428  }
1429 
1430  hwctx->nb_qf = 0;
1431 
1432  /* Pick each queue family to use */
1433 #define PICK_QF(type, vid_op) \
1434  do { \
1435  uint32_t i; \
1436  uint32_t idx; \
1437  \
1438  if (vid_op) \
1439  idx = pick_video_queue_family(qf, qf_vid, num, vid_op); \
1440  else \
1441  idx = pick_queue_family(qf, num, type); \
1442  \
1443  if (idx == -1) \
1444  continue; \
1445  \
1446  for (i = 0; i < hwctx->nb_qf; i++) { \
1447  if (hwctx->qf[i].idx == idx) { \
1448  hwctx->qf[i].flags |= type; \
1449  hwctx->qf[i].video_caps |= vid_op; \
1450  break; \
1451  } \
1452  } \
1453  if (i == hwctx->nb_qf) { \
1454  hwctx->qf[i].idx = idx; \
1455  hwctx->qf[i].num = qf[idx].queueFamilyProperties.queueCount; \
1456  hwctx->qf[i].flags = type; \
1457  hwctx->qf[i].video_caps = vid_op; \
1458  hwctx->nb_qf++; \
1459  } \
1460  } while (0)
1461 
1462  PICK_QF(VK_QUEUE_GRAPHICS_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1463  PICK_QF(VK_QUEUE_COMPUTE_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1464  PICK_QF(VK_QUEUE_TRANSFER_BIT, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1465  PICK_QF(VK_QUEUE_OPTICAL_FLOW_BIT_NV, VK_VIDEO_CODEC_OPERATION_NONE_KHR);
1466 
1467  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H264_BIT_KHR);
1468  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H264_BIT_KHR);
1469 
1470  PICK_QF(VK_QUEUE_VIDEO_ENCODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_ENCODE_H265_BIT_KHR);
1471  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_H265_BIT_KHR);
1472 
1473  PICK_QF(VK_QUEUE_VIDEO_DECODE_BIT_KHR, VK_VIDEO_CODEC_OPERATION_DECODE_AV1_BIT_KHR);
1474 
1475  av_free(qf);
1476  av_free(qf_vid);
1477 
1478 #undef PICK_QF
1479 
1480  cd->pQueueCreateInfos = av_malloc_array(hwctx->nb_qf,
1481  sizeof(VkDeviceQueueCreateInfo));
1482  if (!cd->pQueueCreateInfos)
1483  return AVERROR(ENOMEM);
1484 
1485  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1486  int dup = 0;
1487  float *weights = NULL;
1488  VkDeviceQueueCreateInfo *pc;
1489  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++) {
1490  if (hwctx->qf[i].idx == cd->pQueueCreateInfos[j].queueFamilyIndex) {
1491  dup = 1;
1492  break;
1493  }
1494  }
1495  if (dup)
1496  continue;
1497 
1498  weights = av_malloc_array(hwctx->qf[i].num, sizeof(float));
1499  if (!weights) {
1500  for (uint32_t j = 0; j < cd->queueCreateInfoCount; j++)
1501  av_free((void *)cd->pQueueCreateInfos[i].pQueuePriorities);
1502  av_free((void *)cd->pQueueCreateInfos);
1503  return AVERROR(ENOMEM);
1504  }
1505 
1506  for (uint32_t j = 0; j < hwctx->qf[i].num; j++)
1507  weights[j] = 1.0;
1508 
1509  pc = (VkDeviceQueueCreateInfo *)cd->pQueueCreateInfos;
1510  pc[cd->queueCreateInfoCount++] = (VkDeviceQueueCreateInfo) {
1511  .sType = VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO,
1512  .queueFamilyIndex = hwctx->qf[i].idx,
1513  .queueCount = hwctx->qf[i].num,
1514  .pQueuePriorities = weights,
1515  };
1516  }
1517 
1518 #if FF_API_VULKAN_FIXED_QUEUES
1519 FF_DISABLE_DEPRECATION_WARNINGS
1520  /* Setup deprecated fields */
1521  hwctx->queue_family_index = -1;
1522  hwctx->queue_family_comp_index = -1;
1523  hwctx->queue_family_tx_index = -1;
1524  hwctx->queue_family_encode_index = -1;
1525  hwctx->queue_family_decode_index = -1;
1526 
1527 #define SET_OLD_QF(field, nb_field, type) \
1528  do { \
1529  if (field < 0 && hwctx->qf[i].flags & type) { \
1530  field = hwctx->qf[i].idx; \
1531  nb_field = hwctx->qf[i].num; \
1532  } \
1533  } while (0)
1534 
1535  for (uint32_t i = 0; i < hwctx->nb_qf; i++) {
1536  SET_OLD_QF(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1537  SET_OLD_QF(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1538  SET_OLD_QF(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1539  SET_OLD_QF(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1540  SET_OLD_QF(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1541  }
1542 
1543 #undef SET_OLD_QF
1544 FF_ENABLE_DEPRECATION_WARNINGS
1545 #endif
1546 
1547  return 0;
1548 }
1549 
1550 /* Only resources created by vulkan_device_create should be released here,
1551  * resources created by vulkan_device_init should be released by
1552  * vulkan_device_uninit, to make sure we don't free user provided resources,
1553  * and there is no leak.
1554  */
1555 static void vulkan_device_free(AVHWDeviceContext *ctx)
1556 {
1557  VulkanDevicePriv *p = ctx->hwctx;
1558  AVVulkanDeviceContext *hwctx = &p->p;
1559  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1560 
1561  if (hwctx->act_dev)
1562  vk->DestroyDevice(hwctx->act_dev, hwctx->alloc);
1563 
1564  if (p->debug_ctx)
1565  vk->DestroyDebugUtilsMessengerEXT(hwctx->inst, p->debug_ctx,
1566  hwctx->alloc);
1567 
1568  if (hwctx->inst)
1569  vk->DestroyInstance(hwctx->inst, hwctx->alloc);
1570 
1571  if (p->libvulkan)
1572  dlclose(p->libvulkan);
1573 
1574  RELEASE_PROPS(hwctx->enabled_inst_extensions, hwctx->nb_enabled_inst_extensions);
1575  RELEASE_PROPS(hwctx->enabled_dev_extensions, hwctx->nb_enabled_dev_extensions);
1576 }
1577 
1578 static void vulkan_device_uninit(AVHWDeviceContext *ctx)
1579 {
1580  VulkanDevicePriv *p = ctx->hwctx;
1581 
1582  for (uint32_t i = 0; i < p->nb_tot_qfs; i++) {
1583  pthread_mutex_destroy(p->qf_mutex[i]);
1584  av_freep(&p->qf_mutex[i]);
1585  }
1586  av_freep(&p->qf_mutex);
1587 
1588  ff_vk_uninit(&p->vkctx);
1589 }
1590 
1591 static int vulkan_device_create_internal(AVHWDeviceContext *ctx,
1592  VulkanDeviceSelection *dev_select,
1593  int disable_multiplane,
1594  AVDictionary *opts, int flags)
1595 {
1596  int err = 0;
1597  VkResult ret;
1598  AVDictionaryEntry *opt_d;
1599  VulkanDevicePriv *p = ctx->hwctx;
1600  AVVulkanDeviceContext *hwctx = &p->p;
1601  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1602  enum FFVulkanDebugMode debug_mode = FF_VULKAN_DEBUG_NONE;
1603  VulkanDeviceFeatures supported_feats = { 0 };
1604  VkDeviceCreateInfo dev_info = {
1605  .sType = VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO,
1606  };
1607 
1608  /* Create an instance if not given one */
1609  if ((err = create_instance(ctx, opts, &debug_mode)))
1610  goto end;
1611 
1612  /* Find a physical device (if not given one) */
1613  if ((err = find_device(ctx, dev_select)))
1614  goto end;
1615 
1616  /* Find and enable extensions for the physical device */
1617  if ((err = check_extensions(ctx, 1, opts, &dev_info.ppEnabledExtensionNames,
1618  &dev_info.enabledExtensionCount, debug_mode))) {
1619  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1620  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1621  av_free((void *)dev_info.pQueueCreateInfos);
1622  goto end;
1623  }
1624 
1625  /* Get all supported features for the physical device */
1626  device_features_init(ctx, &supported_feats);
1627  vk->GetPhysicalDeviceFeatures2(hwctx->phys_dev, &supported_feats.device);
1628 
1629  /* Copy all needed features from those supported and activate them */
1630  device_features_init(ctx, &p->feats);
1631  device_features_copy_needed(&p->feats, &supported_feats);
1632  dev_info.pNext = p->feats.device.pNext;
1633  dev_info.pEnabledFeatures = &p->feats.device.features;
1634 
1635  /* Setup enabled queue families */
1636  if ((err = setup_queue_families(ctx, &dev_info)))
1637  goto end;
1638 
1639  /* Finally create the device */
1640  ret = vk->CreateDevice(hwctx->phys_dev, &dev_info, hwctx->alloc,
1641  &hwctx->act_dev);
1642 
1643  for (int i = 0; i < dev_info.queueCreateInfoCount; i++)
1644  av_free((void *)dev_info.pQueueCreateInfos[i].pQueuePriorities);
1645  av_free((void *)dev_info.pQueueCreateInfos);
1646 
1647  if (ret != VK_SUCCESS) {
1648  av_log(ctx, AV_LOG_ERROR, "Device creation failure: %s\n",
1649  ff_vk_ret2str(ret));
1650  for (int i = 0; i < dev_info.enabledExtensionCount; i++)
1651  av_free((void *)dev_info.ppEnabledExtensionNames[i]);
1652  av_free((void *)dev_info.ppEnabledExtensionNames);
1653  err = AVERROR_EXTERNAL;
1654  goto end;
1655  }
1656 
1657  /* Tiled images setting, use them by default */
1658  opt_d = av_dict_get(opts, "linear_images", NULL, 0);
1659  if (opt_d)
1660  p->use_linear_images = strtol(opt_d->value, NULL, 10);
1661 
1662  /* The disable_multiplane argument takes precedent over the option */
1663  p->disable_multiplane = disable_multiplane;
1664  if (!p->disable_multiplane) {
1665  opt_d = av_dict_get(opts, "disable_multiplane", NULL, 0);
1666  if (opt_d)
1667  p->disable_multiplane = strtol(opt_d->value, NULL, 10);
1668  }
1669 
1670  /* Set the public device feature struct and its pNext chain */
1671  hwctx->device_features = p->feats.device;
1672 
1673  /* Set the list of all active extensions */
1674  hwctx->enabled_dev_extensions = dev_info.ppEnabledExtensionNames;
1675  hwctx->nb_enabled_dev_extensions = dev_info.enabledExtensionCount;
1676 
1677  /* The extension lists need to be freed */
1678  ctx->free = vulkan_device_free;
1679 
1680 end:
1681  return err;
1682 }
1683 
1684 static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1685 {
1686  VulkanDevicePriv *p = ctx->hwctx;
1687  pthread_mutex_lock(&p->qf_mutex[queue_family][index]);
1688 }
1689 
1690 static void unlock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
1691 {
1692  VulkanDevicePriv *p = ctx->hwctx;
1693  pthread_mutex_unlock(&p->qf_mutex[queue_family][index]);
1694 }
1695 
1696 static int vulkan_device_init(AVHWDeviceContext *ctx)
1697 {
1698  int err = 0;
1699  uint32_t qf_num;
1700  VulkanDevicePriv *p = ctx->hwctx;
1701  AVVulkanDeviceContext *hwctx = &p->p;
1702  FFVulkanFunctions *vk = &p->vkctx.vkfn;
1703  VkQueueFamilyProperties2 *qf;
1704  VkQueueFamilyVideoPropertiesKHR *qf_vid;
1705  int graph_index, comp_index, tx_index, enc_index, dec_index;
1706 
1707  /* Set device extension flags */
1708  for (int i = 0; i < hwctx->nb_enabled_dev_extensions; i++) {
1709  for (int j = 0; j < FF_ARRAY_ELEMS(optional_device_exts); j++) {
1710  if (!strcmp(hwctx->enabled_dev_extensions[i],
1711  optional_device_exts[j].name)) {
1712  p->vkctx.extensions |= optional_device_exts[j].flag;
1713  break;
1714  }
1715  }
1716  }
1717 
1718  err = ff_vk_load_functions(ctx, vk, p->vkctx.extensions, 1, 1);
1719  if (err < 0) {
1720  av_log(ctx, AV_LOG_ERROR, "Unable to load functions!\n");
1721  return err;
1722  }
1723 
1724  p->props.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_PROPERTIES_2;
1725  p->props.pNext = &p->hprops;
1726  p->hprops.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_MEMORY_HOST_PROPERTIES_EXT;
1727 
1728  vk->GetPhysicalDeviceProperties2(hwctx->phys_dev, &p->props);
1729  av_log(ctx, AV_LOG_VERBOSE, "Using device: %s\n",
1730  p->props.properties.deviceName);
1731  av_log(ctx, AV_LOG_VERBOSE, "Alignments:\n");
1732  av_log(ctx, AV_LOG_VERBOSE, " optimalBufferCopyRowPitchAlignment: %"PRIu64"\n",
1733  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
1734  av_log(ctx, AV_LOG_VERBOSE, " minMemoryMapAlignment: %"SIZE_SPECIFIER"\n",
1735  p->props.properties.limits.minMemoryMapAlignment);
1736  av_log(ctx, AV_LOG_VERBOSE, " nonCoherentAtomSize: %"PRIu64"\n",
1737  p->props.properties.limits.nonCoherentAtomSize);
1738  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY)
1739  av_log(ctx, AV_LOG_VERBOSE, " minImportedHostPointerAlignment: %"PRIu64"\n",
1740  p->hprops.minImportedHostPointerAlignment);
1741 
1742  p->dev_is_nvidia = (p->props.properties.vendorID == 0x10de);
1743 
1744  vk->GetPhysicalDeviceQueueFamilyProperties(hwctx->phys_dev, &qf_num, NULL);
1745  if (!qf_num) {
1746  av_log(ctx, AV_LOG_ERROR, "Failed to get queues!\n");
1747  return AVERROR_EXTERNAL;
1748  }
1749 
1750  qf = av_malloc_array(qf_num, sizeof(VkQueueFamilyProperties2));
1751  if (!qf)
1752  return AVERROR(ENOMEM);
1753 
1754  qf_vid = av_malloc_array(qf_num, sizeof(VkQueueFamilyVideoPropertiesKHR));
1755  if (!qf_vid) {
1756  av_free(qf);
1757  return AVERROR(ENOMEM);
1758  }
1759 
1760  for (uint32_t i = 0; i < qf_num; i++) {
1761  qf_vid[i] = (VkQueueFamilyVideoPropertiesKHR) {
1762  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_VIDEO_PROPERTIES_KHR,
1763  };
1764  qf[i] = (VkQueueFamilyProperties2) {
1765  .sType = VK_STRUCTURE_TYPE_QUEUE_FAMILY_PROPERTIES_2,
1766  .pNext = &qf_vid[i],
1767  };
1768  }
1769 
1770  vk->GetPhysicalDeviceQueueFamilyProperties2(hwctx->phys_dev, &qf_num, qf);
1771 
1772  p->qf_mutex = av_calloc(qf_num, sizeof(*p->qf_mutex));
1773  if (!p->qf_mutex) {
1774  err = AVERROR(ENOMEM);
1775  goto end;
1776  }
1777  p->nb_tot_qfs = qf_num;
1778 
1779  for (uint32_t i = 0; i < qf_num; i++) {
1780  p->qf_mutex[i] = av_calloc(qf[i].queueFamilyProperties.queueCount,
1781  sizeof(**p->qf_mutex));
1782  if (!p->qf_mutex[i]) {
1783  err = AVERROR(ENOMEM);
1784  goto end;
1785  }
1786  for (uint32_t j = 0; j < qf[i].queueFamilyProperties.queueCount; j++) {
1787  err = pthread_mutex_init(&p->qf_mutex[i][j], NULL);
1788  if (err != 0) {
1789  av_log(ctx, AV_LOG_ERROR, "pthread_mutex_init failed : %s\n",
1790  av_err2str(err));
1791  err = AVERROR(err);
1792  goto end;
1793  }
1794  }
1795  }
1796 
1797 #if FF_API_VULKAN_FIXED_QUEUES
1798 FF_DISABLE_DEPRECATION_WARNINGS
1799  graph_index = hwctx->nb_graphics_queues ? hwctx->queue_family_index : -1;
1800  comp_index = hwctx->nb_comp_queues ? hwctx->queue_family_comp_index : -1;
1801  tx_index = hwctx->nb_tx_queues ? hwctx->queue_family_tx_index : -1;
1802  dec_index = hwctx->nb_decode_queues ? hwctx->queue_family_decode_index : -1;
1803  enc_index = hwctx->nb_encode_queues ? hwctx->queue_family_encode_index : -1;
1804 
1805 #define CHECK_QUEUE(type, required, fidx, ctx_qf, qc) \
1806  do { \
1807  if (ctx_qf < 0 && required) { \
1808  av_log(ctx, AV_LOG_ERROR, "%s queue family is required, but marked as missing" \
1809  " in the context!\n", type); \
1810  err = AVERROR(EINVAL); \
1811  goto end; \
1812  } else if (fidx < 0 || ctx_qf < 0) { \
1813  break; \
1814  } else if (ctx_qf >= qf_num) { \
1815  av_log(ctx, AV_LOG_ERROR, "Invalid %s family index %i (device has %i families)!\n", \
1816  type, ctx_qf, qf_num); \
1817  err = AVERROR(EINVAL); \
1818  goto end; \
1819  } \
1820  \
1821  av_log(ctx, AV_LOG_VERBOSE, "Using queue family %i (queues: %i)" \
1822  " for%s%s%s%s%s\n", \
1823  ctx_qf, qc, \
1824  ctx_qf == graph_index ? " graphics" : "", \
1825  ctx_qf == comp_index ? " compute" : "", \
1826  ctx_qf == tx_index ? " transfers" : "", \
1827  ctx_qf == enc_index ? " encode" : "", \
1828  ctx_qf == dec_index ? " decode" : ""); \
1829  graph_index = (ctx_qf == graph_index) ? -1 : graph_index; \
1830  comp_index = (ctx_qf == comp_index) ? -1 : comp_index; \
1831  tx_index = (ctx_qf == tx_index) ? -1 : tx_index; \
1832  enc_index = (ctx_qf == enc_index) ? -1 : enc_index; \
1833  dec_index = (ctx_qf == dec_index) ? -1 : dec_index; \
1834  } while (0)
1835 
1836  CHECK_QUEUE("graphics", 0, graph_index, hwctx->queue_family_index, hwctx->nb_graphics_queues);
1837  CHECK_QUEUE("compute", 1, comp_index, hwctx->queue_family_comp_index, hwctx->nb_comp_queues);
1838  CHECK_QUEUE("upload", 1, tx_index, hwctx->queue_family_tx_index, hwctx->nb_tx_queues);
1839  CHECK_QUEUE("decode", 0, dec_index, hwctx->queue_family_decode_index, hwctx->nb_decode_queues);
1840  CHECK_QUEUE("encode", 0, enc_index, hwctx->queue_family_encode_index, hwctx->nb_encode_queues);
1841 
1842 #undef CHECK_QUEUE
1843 
1844  /* Update the new queue family fields. If non-zero already,
1845  * it means API users have set it. */
1846  if (!hwctx->nb_qf) {
1847 #define ADD_QUEUE(ctx_qf, qc, flag) \
1848  do { \
1849  if (ctx_qf != -1) { \
1850  hwctx->qf[hwctx->nb_qf++] = (AVVulkanDeviceQueueFamily) { \
1851  .idx = ctx_qf, \
1852  .num = qc, \
1853  .flags = flag, \
1854  }; \
1855  } \
1856  } while (0)
1857 
1858  ADD_QUEUE(hwctx->queue_family_index, hwctx->nb_graphics_queues, VK_QUEUE_GRAPHICS_BIT);
1859  ADD_QUEUE(hwctx->queue_family_comp_index, hwctx->nb_comp_queues, VK_QUEUE_COMPUTE_BIT);
1860  ADD_QUEUE(hwctx->queue_family_tx_index, hwctx->nb_tx_queues, VK_QUEUE_TRANSFER_BIT);
1861  ADD_QUEUE(hwctx->queue_family_decode_index, hwctx->nb_decode_queues, VK_QUEUE_VIDEO_DECODE_BIT_KHR);
1862  ADD_QUEUE(hwctx->queue_family_encode_index, hwctx->nb_encode_queues, VK_QUEUE_VIDEO_ENCODE_BIT_KHR);
1863 #undef ADD_QUEUE
1864  }
1865 FF_ENABLE_DEPRECATION_WARNINGS
1866 #endif
1867 
1868  for (int i = 0; i < hwctx->nb_qf; i++) {
1869  if (!hwctx->qf[i].video_caps &&
1870  hwctx->qf[i].flags & (VK_QUEUE_VIDEO_DECODE_BIT_KHR |
1871  VK_QUEUE_VIDEO_ENCODE_BIT_KHR)) {
1872  hwctx->qf[i].video_caps = qf_vid[hwctx->qf[i].idx].videoCodecOperations;
1873  }
1874  }
1875 
1876  /* Setup array for pQueueFamilyIndices with used queue families */
1877  p->nb_img_qfs = 0;
1878  for (int i = 0; i < hwctx->nb_qf; i++) {
1879  int seen = 0;
1880  /* Make sure each entry is unique
1881  * (VUID-VkBufferCreateInfo-sharingMode-01419) */
1882  for (int j = (i - 1); j >= 0; j--) {
1883  if (hwctx->qf[i].idx == hwctx->qf[j].idx) {
1884  seen = 1;
1885  break;
1886  }
1887  }
1888  if (!seen)
1889  p->img_qfs[p->nb_img_qfs++] = hwctx->qf[i].idx;
1890  }
1891 
1892  if (!hwctx->lock_queue)
1893  hwctx->lock_queue = lock_queue;
1894  if (!hwctx->unlock_queue)
1895  hwctx->unlock_queue = unlock_queue;
1896 
1897  /* Get device capabilities */
1898  vk->GetPhysicalDeviceMemoryProperties(hwctx->phys_dev, &p->mprops);
1899 
1900  p->vkctx.device = ctx;
1901  p->vkctx.hwctx = hwctx;
1902 
1903  ff_vk_load_props(&p->vkctx);
1904  ff_vk_qf_init(&p->vkctx, &p->compute_qf, VK_QUEUE_COMPUTE_BIT);
1905  ff_vk_qf_init(&p->vkctx, &p->transfer_qf, VK_QUEUE_TRANSFER_BIT);
1906 
1907 end:
1908  av_free(qf_vid);
1909  av_free(qf);
1910  return err;
1911 }
1912 
1913 static int vulkan_device_create(AVHWDeviceContext *ctx, const char *device,
1914  AVDictionary *opts, int flags)
1915 {
1916  VulkanDeviceSelection dev_select = { 0 };
1917  if (device && device[0]) {
1918  char *end = NULL;
1919  dev_select.index = strtol(device, &end, 10);
1920  if (end == device) {
1921  dev_select.index = 0;
1922  dev_select.name = device;
1923  }
1924  }
1925 
1926  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
1927 }
1928 
1929 static int vulkan_device_derive(AVHWDeviceContext *ctx,
1930  AVHWDeviceContext *src_ctx,
1931  AVDictionary *opts, int flags)
1932 {
1933  av_unused VulkanDeviceSelection dev_select = { 0 };
1934 
1935  /* If there's only one device on the system, then even if its not covered
1936  * by the following checks (e.g. non-PCIe ARM GPU), having an empty
1937  * dev_select will mean it'll get picked. */
1938  switch(src_ctx->type) {
1939 #if CONFIG_VAAPI
1940  case AV_HWDEVICE_TYPE_VAAPI: {
1941  AVVAAPIDeviceContext *src_hwctx = src_ctx->hwctx;
1942  VADisplay dpy = src_hwctx->display;
1943 #if VA_CHECK_VERSION(1, 15, 0)
1944  VAStatus vas;
1945  VADisplayAttribute attr = {
1946  .type = VADisplayPCIID,
1947  };
1948 #endif
1949  const char *vendor;
1950 
1951 #if VA_CHECK_VERSION(1, 15, 0)
1952  vas = vaGetDisplayAttributes(dpy, &attr, 1);
1953  if (vas == VA_STATUS_SUCCESS && attr.flags != VA_DISPLAY_ATTRIB_NOT_SUPPORTED)
1954  dev_select.pci_device = (attr.value & 0xFFFF);
1955 #endif
1956 
1957  if (!dev_select.pci_device) {
1958  vendor = vaQueryVendorString(dpy);
1959  if (!vendor) {
1960  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from VAAPI!\n");
1961  return AVERROR_EXTERNAL;
1962  }
1963 
1964  if (strstr(vendor, "AMD"))
1965  dev_select.vendor_id = 0x1002;
1966  }
1967 
1968  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
1969  }
1970 #endif
1971 #if CONFIG_LIBDRM
1972  case AV_HWDEVICE_TYPE_DRM: {
1973  int err;
1974  struct stat drm_node_info;
1975  drmDevice *drm_dev_info;
1976  AVDRMDeviceContext *src_hwctx = src_ctx->hwctx;
1977 
1978  err = fstat(src_hwctx->fd, &drm_node_info);
1979  if (err) {
1980  av_log(ctx, AV_LOG_ERROR, "Unable to get node info from DRM fd: %s!\n",
1981  av_err2str(AVERROR(errno)));
1982  return AVERROR_EXTERNAL;
1983  }
1984 
1985  dev_select.drm_major = major(drm_node_info.st_dev);
1986  dev_select.drm_minor = minor(drm_node_info.st_dev);
1987  dev_select.has_drm = 1;
1988 
1989  err = drmGetDevice(src_hwctx->fd, &drm_dev_info);
1990  if (err) {
1991  av_log(ctx, AV_LOG_ERROR, "Unable to get device info from DRM fd: %s!\n",
1992  av_err2str(AVERROR(errno)));
1993  return AVERROR_EXTERNAL;
1994  }
1995 
1996  if (drm_dev_info->bustype == DRM_BUS_PCI)
1997  dev_select.pci_device = drm_dev_info->deviceinfo.pci->device_id;
1998 
1999  drmFreeDevice(&drm_dev_info);
2000 
2001  return vulkan_device_create_internal(ctx, &dev_select, 0, opts, flags);
2002  }
2003 #endif
2004 #if CONFIG_CUDA
2005  case AV_HWDEVICE_TYPE_CUDA: {
2006  AVHWDeviceContext *cuda_cu = src_ctx;
2007  AVCUDADeviceContext *src_hwctx = src_ctx->hwctx;
2008  AVCUDADeviceContextInternal *cu_internal = src_hwctx->internal;
2009  CudaFunctions *cu = cu_internal->cuda_dl;
2010 
2011  int ret = CHECK_CU(cu->cuDeviceGetUuid((CUuuid *)&dev_select.uuid,
2012  cu_internal->cuda_device));
2013  if (ret < 0) {
2014  av_log(ctx, AV_LOG_ERROR, "Unable to get UUID from CUDA!\n");
2015  return AVERROR_EXTERNAL;
2016  }
2017 
2018  dev_select.has_uuid = 1;
2019 
2020  /*
2021  * CUDA is not able to import multiplane images, so always derive a
2022  * Vulkan device with multiplane disabled.
2023  */
2024  return vulkan_device_create_internal(ctx, &dev_select, 1, opts, flags);
2025  }
2026 #endif
2027  default:
2028  return AVERROR(ENOSYS);
2029  }
2030 }
2031 
2032 static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx,
2033  const void *hwconfig,
2034  AVHWFramesConstraints *constraints)
2035 {
2036  int count = 0;
2037  VulkanDevicePriv *p = ctx->hwctx;
2038 
2039  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2040  count += vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2041  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2042  VK_IMAGE_TILING_OPTIMAL,
2043  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0;
2044  }
2045 
2046  constraints->valid_sw_formats = av_malloc_array(count + 1,
2047  sizeof(enum AVPixelFormat));
2048  if (!constraints->valid_sw_formats)
2049  return AVERROR(ENOMEM);
2050 
2051  count = 0;
2052  for (enum AVPixelFormat i = 0; i < nb_vk_formats_list; i++) {
2053  if (vkfmt_from_pixfmt2(ctx, vk_formats_list[i].pixfmt,
2054  p->use_linear_images ? VK_IMAGE_TILING_LINEAR :
2055  VK_IMAGE_TILING_OPTIMAL,
2056  NULL, NULL, NULL, NULL, p->disable_multiplane, 1) >= 0) {
2057  constraints->valid_sw_formats[count++] = vk_formats_list[i].pixfmt;
2058  }
2059  }
2060 
2061  constraints->valid_sw_formats[count++] = AV_PIX_FMT_NONE;
2062 
2063  constraints->min_width = 1;
2064  constraints->min_height = 1;
2065  constraints->max_width = p->props.properties.limits.maxImageDimension2D;
2066  constraints->max_height = p->props.properties.limits.maxImageDimension2D;
2067 
2068  constraints->valid_hw_formats = av_malloc_array(2, sizeof(enum AVPixelFormat));
2069  if (!constraints->valid_hw_formats)
2070  return AVERROR(ENOMEM);
2071 
2072  constraints->valid_hw_formats[0] = AV_PIX_FMT_VULKAN;
2073  constraints->valid_hw_formats[1] = AV_PIX_FMT_NONE;
2074 
2075  return 0;
2076 }
2077 
2078 static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req,
2079  VkMemoryPropertyFlagBits req_flags, const void *alloc_extension,
2080  VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
2081 {
2082  VkResult ret;
2083  int index = -1;
2084  VulkanDevicePriv *p = ctx->hwctx;
2085  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2086  AVVulkanDeviceContext *dev_hwctx = &p->p;
2087  VkMemoryAllocateInfo alloc_info = {
2088  .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
2089  .pNext = alloc_extension,
2090  .allocationSize = req->size,
2091  };
2092 
2093  /* The vulkan spec requires memory types to be sorted in the "optimal"
2094  * order, so the first matching type we find will be the best/fastest one */
2095  for (int i = 0; i < p->mprops.memoryTypeCount; i++) {
2096  const VkMemoryType *type = &p->mprops.memoryTypes[i];
2097 
2098  /* The memory type must be supported by the requirements (bitfield) */
2099  if (!(req->memoryTypeBits & (1 << i)))
2100  continue;
2101 
2102  /* The memory type flags must include our properties */
2103  if ((type->propertyFlags & req_flags) != req_flags)
2104  continue;
2105 
2106  /* The memory type must be large enough */
2107  if (req->size > p->mprops.memoryHeaps[type->heapIndex].size)
2108  continue;
2109 
2110  /* Found a suitable memory type */
2111  index = i;
2112  break;
2113  }
2114 
2115  if (index < 0) {
2116  av_log(ctx, AV_LOG_ERROR, "No memory type found for flags 0x%x\n",
2117  req_flags);
2118  return AVERROR(EINVAL);
2119  }
2120 
2121  alloc_info.memoryTypeIndex = index;
2122 
2123  ret = vk->AllocateMemory(dev_hwctx->act_dev, &alloc_info,
2124  dev_hwctx->alloc, mem);
2125  if (ret != VK_SUCCESS) {
2126  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory: %s\n",
2127  ff_vk_ret2str(ret));
2128  return AVERROR(ENOMEM);
2129  }
2130 
2131  *mem_flags |= p->mprops.memoryTypes[index].propertyFlags;
2132 
2133  return 0;
2134 }
2135 
2136 static void vulkan_free_internal(AVVkFrame *f)
2137 {
2138  av_unused AVVkFrameInternal *internal = f->internal;
2139 
2140 #if CONFIG_CUDA
2141  if (internal->cuda_fc_ref) {
2142  AVHWFramesContext *cuda_fc = (AVHWFramesContext *)internal->cuda_fc_ref->data;
2143  int planes = av_pix_fmt_count_planes(cuda_fc->sw_format);
2144  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
2145  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
2146  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
2147  CudaFunctions *cu = cu_internal->cuda_dl;
2148 
2149  for (int i = 0; i < planes; i++) {
2150  if (internal->cu_sem[i])
2151  CHECK_CU(cu->cuDestroyExternalSemaphore(internal->cu_sem[i]));
2152  if (internal->cu_mma[i])
2153  CHECK_CU(cu->cuMipmappedArrayDestroy(internal->cu_mma[i]));
2154  if (internal->ext_mem[i])
2155  CHECK_CU(cu->cuDestroyExternalMemory(internal->ext_mem[i]));
2156 #ifdef _WIN32
2157  if (internal->ext_sem_handle[i])
2158  CloseHandle(internal->ext_sem_handle[i]);
2159  if (internal->ext_mem_handle[i])
2160  CloseHandle(internal->ext_mem_handle[i]);
2161 #endif
2162  }
2163 
2164  av_buffer_unref(&internal->cuda_fc_ref);
2165  }
2166 #endif
2167 
2168  pthread_mutex_destroy(&internal->update_mutex);
2169  av_freep(&f->internal);
2170 }
2171 
2172 static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
2173 {
2174  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2175  AVVulkanDeviceContext *hwctx = &p->p;
2176  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2177  int nb_images = ff_vk_count_images(f);
2178  int nb_sems = 0;
2179 
2180  while (nb_sems < FF_ARRAY_ELEMS(f->sem) && f->sem[nb_sems])
2181  nb_sems++;
2182 
2183  if (nb_sems) {
2184  VkSemaphoreWaitInfo sem_wait = {
2185  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
2186  .flags = 0x0,
2187  .pSemaphores = f->sem,
2188  .pValues = f->sem_value,
2189  .semaphoreCount = nb_sems,
2190  };
2191 
2192  vk->WaitSemaphores(hwctx->act_dev, &sem_wait, UINT64_MAX);
2193  }
2194 
2195  vulkan_free_internal(f);
2196 
2197  for (int i = 0; i < nb_images; i++) {
2198  vk->DestroyImage(hwctx->act_dev, f->img[i], hwctx->alloc);
2199  vk->FreeMemory(hwctx->act_dev, f->mem[i], hwctx->alloc);
2200  vk->DestroySemaphore(hwctx->act_dev, f->sem[i], hwctx->alloc);
2201  }
2202 
2203  av_free(f);
2204 }
2205 
2206 static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
2207 {
2208  vulkan_frame_free(opaque, (AVVkFrame*)data);
2209 }
2210 
2211 static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f,
2212  void *alloc_pnext, size_t alloc_pnext_stride)
2213 {
2214  int img_cnt = 0, err;
2215  VkResult ret;
2216  AVHWDeviceContext *ctx = hwfc->device_ctx;
2217  VulkanDevicePriv *p = ctx->hwctx;
2218  AVVulkanDeviceContext *hwctx = &p->p;
2219  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2220  VkBindImageMemoryInfo bind_info[AV_NUM_DATA_POINTERS] = { { 0 } };
2221 
2222  while (f->img[img_cnt]) {
2223  int use_ded_mem;
2224  VkImageMemoryRequirementsInfo2 req_desc = {
2225  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
2226  .image = f->img[img_cnt],
2227  };
2228  VkMemoryDedicatedAllocateInfo ded_alloc = {
2229  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
2230  .pNext = (void *)(((uint8_t *)alloc_pnext) + img_cnt*alloc_pnext_stride),
2231  };
2232  VkMemoryDedicatedRequirements ded_req = {
2233  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
2234  };
2235  VkMemoryRequirements2 req = {
2236  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
2237  .pNext = &ded_req,
2238  };
2239 
2240  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req);
2241 
2242  if (f->tiling == VK_IMAGE_TILING_LINEAR)
2243  req.memoryRequirements.size = FFALIGN(req.memoryRequirements.size,
2244  p->props.properties.limits.minMemoryMapAlignment);
2245 
2246  /* In case the implementation prefers/requires dedicated allocation */
2247  use_ded_mem = ded_req.prefersDedicatedAllocation |
2248  ded_req.requiresDedicatedAllocation;
2249  if (use_ded_mem)
2250  ded_alloc.image = f->img[img_cnt];
2251 
2252  /* Allocate memory */
2253  if ((err = alloc_mem(ctx, &req.memoryRequirements,
2254  f->tiling == VK_IMAGE_TILING_LINEAR ?
2255  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT :
2256  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
2257  use_ded_mem ? &ded_alloc : (void *)ded_alloc.pNext,
2258  &f->flags, &f->mem[img_cnt])))
2259  return err;
2260 
2261  f->size[img_cnt] = req.memoryRequirements.size;
2262  bind_info[img_cnt].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
2263  bind_info[img_cnt].image = f->img[img_cnt];
2264  bind_info[img_cnt].memory = f->mem[img_cnt];
2265 
2266  img_cnt++;
2267  }
2268 
2269  /* Bind the allocated memory to the images */
2270  ret = vk->BindImageMemory2(hwctx->act_dev, img_cnt, bind_info);
2271  if (ret != VK_SUCCESS) {
2272  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
2273  ff_vk_ret2str(ret));
2274  return AVERROR_EXTERNAL;
2275  }
2276 
2277  return 0;
2278 }
2279 
2280 enum PrepMode {
2281  PREP_MODE_GENERAL,
2282  PREP_MODE_WRITE,
2283  PREP_MODE_EXTERNAL_EXPORT,
2284  PREP_MODE_EXTERNAL_IMPORT,
2285  PREP_MODE_DECODING_DST,
2286  PREP_MODE_DECODING_DPB,
2287  PREP_MODE_ENCODING_DPB,
2288 };
2289 
2290 static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx,
2291  AVVkFrame *frame, enum PrepMode pmode)
2292 {
2293  int err;
2294  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2295  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2296  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
2297  int nb_img_bar = 0;
2298 
2299  uint32_t dst_qf = VK_QUEUE_FAMILY_IGNORED;
2300  VkImageLayout new_layout;
2301  VkAccessFlags2 new_access;
2302  VkPipelineStageFlagBits2 src_stage = VK_PIPELINE_STAGE_2_NONE;
2303 
2304  /* This is dirty - but it works. The vulkan.c dependency system doesn't
2305  * free non-refcounted frames, and non-refcounted hardware frames cannot
2306  * happen anywhere outside of here. */
2307  AVBufferRef tmp_ref = {
2308  .data = (uint8_t *)hwfc,
2309  };
2310  AVFrame tmp_frame = {
2311  .data[0] = (uint8_t *)frame,
2312  .hw_frames_ctx = &tmp_ref,
2313  };
2314 
2315  VkCommandBuffer cmd_buf;
2316  FFVkExecContext *exec = ff_vk_exec_get(&p->vkctx, ectx);
2317  cmd_buf = exec->buf;
2318  ff_vk_exec_start(&p->vkctx, exec);
2319 
2320  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, &tmp_frame,
2321  VK_PIPELINE_STAGE_2_NONE,
2322  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
2323  if (err < 0)
2324  return err;
2325 
2326  switch (pmode) {
2327  case PREP_MODE_GENERAL:
2328  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2329  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2330  break;
2331  case PREP_MODE_WRITE:
2332  new_layout = VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
2333  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2334  break;
2335  case PREP_MODE_EXTERNAL_IMPORT:
2336  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2337  new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2338  break;
2339  case PREP_MODE_EXTERNAL_EXPORT:
2340  new_layout = VK_IMAGE_LAYOUT_GENERAL;
2341  new_access = VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT;
2342  dst_qf = VK_QUEUE_FAMILY_EXTERNAL_KHR;
2343  src_stage = VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT;
2344  break;
2345  case PREP_MODE_DECODING_DST:
2346  new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DST_KHR;
2347  new_access = VK_ACCESS_TRANSFER_WRITE_BIT;
2348  break;
2349  case PREP_MODE_DECODING_DPB:
2350  new_layout = VK_IMAGE_LAYOUT_VIDEO_DECODE_DPB_KHR;
2351  new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2352  break;
2353  case PREP_MODE_ENCODING_DPB:
2354  new_layout = VK_IMAGE_LAYOUT_VIDEO_ENCODE_DPB_KHR;
2355  new_access = VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2356  break;
2357  }
2358 
2359  ff_vk_frame_barrier(&p->vkctx, exec, &tmp_frame, img_bar, &nb_img_bar,
2360  src_stage,
2361  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
2362  new_access, new_layout, dst_qf);
2363 
2364  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
2365  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
2366  .pImageMemoryBarriers = img_bar,
2367  .imageMemoryBarrierCount = nb_img_bar,
2368  });
2369 
2370  err = ff_vk_exec_submit(&p->vkctx, exec);
2371  if (err < 0)
2372  return err;
2373 
2374  /* We can do this because there are no real dependencies */
2375  ff_vk_exec_discard_deps(&p->vkctx, exec);
2376 
2377  return 0;
2378 }
2379 
2380 static inline void get_plane_wh(uint32_t *w, uint32_t *h, enum AVPixelFormat format,
2381  int frame_w, int frame_h, int plane)
2382 {
2383  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(format);
2384 
2385  /* Currently always true unless gray + alpha support is added */
2386  if (!plane || (plane == 3) || desc->flags & AV_PIX_FMT_FLAG_RGB ||
2387  !(desc->flags & AV_PIX_FMT_FLAG_PLANAR)) {
2388  *w = frame_w;
2389  *h = frame_h;
2390  return;
2391  }
2392 
2393  *w = AV_CEIL_RSHIFT(frame_w, desc->log2_chroma_w);
2394  *h = AV_CEIL_RSHIFT(frame_h, desc->log2_chroma_h);
2395 }
2396 
2397 static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame,
2398  VkImageTiling tiling, VkImageUsageFlagBits usage,
2399  VkImageCreateFlags flags, int nb_layers,
2400  void *create_pnext)
2401 {
2402  int err;
2403  VkResult ret;
2404  AVVulkanFramesContext *hwfc_vk = hwfc->hwctx;
2405  AVHWDeviceContext *ctx = hwfc->device_ctx;
2406  VulkanDevicePriv *p = ctx->hwctx;
2407  AVVulkanDeviceContext *hwctx = &p->p;
2408  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2409 
2410  VkExportSemaphoreCreateInfo ext_sem_info = {
2411  .sType = VK_STRUCTURE_TYPE_EXPORT_SEMAPHORE_CREATE_INFO,
2412 #ifdef _WIN32
2413  .handleTypes = IsWindows8OrGreater()
2414  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
2415  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
2416 #else
2417  .handleTypes = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
2418 #endif
2419  };
2420 
2421  VkSemaphoreTypeCreateInfo sem_type_info = {
2422  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2423 #ifdef _WIN32
2424  .pNext = p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM ? &ext_sem_info : NULL,
2425 #else
2426  .pNext = p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM ? &ext_sem_info : NULL,
2427 #endif
2428  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2429  .initialValue = 0,
2430  };
2431 
2432  VkSemaphoreCreateInfo sem_spawn = {
2433  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2434  .pNext = &sem_type_info,
2435  };
2436 
2437  AVVkFrame *f = av_vk_frame_alloc();
2438  if (!f) {
2439  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2440  return AVERROR(ENOMEM);
2441  }
2442 
2443  // TODO: check witdh and height for alignment in case of multiplanar (must be mod-2 if subsampled)
2444 
2445  /* Create the images */
2446  for (int i = 0; (hwfc_vk->format[i] != VK_FORMAT_UNDEFINED); i++) {
2447  VkImageCreateInfo create_info = {
2448  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2449  .pNext = create_pnext,
2450  .imageType = VK_IMAGE_TYPE_2D,
2451  .format = hwfc_vk->format[i],
2452  .extent.depth = 1,
2453  .mipLevels = 1,
2454  .arrayLayers = nb_layers,
2455  .flags = flags,
2456  .tiling = tiling,
2457  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED,
2458  .usage = usage,
2459  .samples = VK_SAMPLE_COUNT_1_BIT,
2460  .pQueueFamilyIndices = p->img_qfs,
2461  .queueFamilyIndexCount = p->nb_img_qfs,
2462  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2463  VK_SHARING_MODE_EXCLUSIVE,
2464  };
2465 
2466  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
2467  hwfc->sw_format, hwfc->width, hwfc->height, i);
2468 
2469  ret = vk->CreateImage(hwctx->act_dev, &create_info,
2470  hwctx->alloc, &f->img[i]);
2471  if (ret != VK_SUCCESS) {
2472  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
2473  ff_vk_ret2str(ret));
2474  err = AVERROR(EINVAL);
2475  goto fail;
2476  }
2477 
2478  /* Create semaphore */
2479  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
2480  hwctx->alloc, &f->sem[i]);
2481  if (ret != VK_SUCCESS) {
2482  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
2483  ff_vk_ret2str(ret));
2484  err = AVERROR_EXTERNAL;
2485  goto fail;
2486  }
2487 
2488  f->queue_family[i] = p->nb_img_qfs > 1 ? VK_QUEUE_FAMILY_IGNORED : p->img_qfs[0];
2489  f->layout[i] = create_info.initialLayout;
2490  f->access[i] = 0x0;
2491  f->sem_value[i] = 0;
2492  }
2493 
2494  f->flags = 0x0;
2495  f->tiling = tiling;
2496 
2497  *frame = f;
2498  return 0;
2499 
2500 fail:
2501  vulkan_frame_free(hwfc, f);
2502  return err;
2503 }
2504 
2505 /* Checks if an export flag is enabled, and if it is ORs it with *iexp */
2506 static void try_export_flags(AVHWFramesContext *hwfc,
2507  VkExternalMemoryHandleTypeFlags *comp_handle_types,
2508  VkExternalMemoryHandleTypeFlagBits *iexp,
2509  VkExternalMemoryHandleTypeFlagBits exp)
2510 {
2511  VkResult ret;
2512  AVVulkanFramesContext *hwctx = hwfc->hwctx;
2513  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2514  AVVulkanDeviceContext *dev_hwctx = &p->p;
2515  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2516 
2517  const VkImageDrmFormatModifierListCreateInfoEXT *drm_mod_info =
2518  ff_vk_find_struct(hwctx->create_pnext,
2519  VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_LIST_CREATE_INFO_EXT);
2520  int has_mods = hwctx->tiling == VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT && drm_mod_info;
2521  int nb_mods;
2522 
2523  VkExternalImageFormatProperties eprops = {
2524  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2525  };
2526  VkImageFormatProperties2 props = {
2527  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2528  .pNext = &eprops,
2529  };
2530  VkPhysicalDeviceImageDrmFormatModifierInfoEXT phy_dev_mod_info = {
2531  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2532  .pNext = NULL,
2533  .pQueueFamilyIndices = p->img_qfs,
2534  .queueFamilyIndexCount = p->nb_img_qfs,
2535  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2536  VK_SHARING_MODE_EXCLUSIVE,
2537  };
2538  VkPhysicalDeviceExternalImageFormatInfo enext = {
2539  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
2540  .handleType = exp,
2541  .pNext = has_mods ? &phy_dev_mod_info : NULL,
2542  };
2543  VkPhysicalDeviceImageFormatInfo2 pinfo = {
2544  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
2545  .pNext = !exp ? NULL : &enext,
2546  .format = av_vkfmt_from_pixfmt(hwfc->sw_format)[0],
2547  .type = VK_IMAGE_TYPE_2D,
2548  .tiling = hwctx->tiling,
2549  .usage = hwctx->usage,
2550  .flags = VK_IMAGE_CREATE_ALIAS_BIT,
2551  };
2552 
2553  nb_mods = has_mods ? drm_mod_info->drmFormatModifierCount : 1;
2554  for (int i = 0; i < nb_mods; i++) {
2555  if (has_mods)
2556  phy_dev_mod_info.drmFormatModifier = drm_mod_info->pDrmFormatModifiers[i];
2557 
2558  ret = vk->GetPhysicalDeviceImageFormatProperties2(dev_hwctx->phys_dev,
2559  &pinfo, &props);
2560 
2561  if (ret == VK_SUCCESS) {
2562  *iexp |= exp;
2563  *comp_handle_types |= eprops.externalMemoryProperties.compatibleHandleTypes;
2564  }
2565  }
2566 }
2567 
2568 static AVBufferRef *vulkan_pool_alloc(void *opaque, size_t size)
2569 {
2570  int err;
2571  AVVkFrame *f;
2572  AVBufferRef *avbuf = NULL;
2573  AVHWFramesContext *hwfc = opaque;
2574  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2575  VulkanFramesPriv *fp = hwfc->hwctx;
2576  AVVulkanFramesContext *hwctx = &fp->p;
2577  VkExternalMemoryHandleTypeFlags e = 0x0;
2578  VkExportMemoryAllocateInfo eminfo[AV_NUM_DATA_POINTERS];
2579 
2580  VkExternalMemoryImageCreateInfo eiinfo = {
2581  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2582  .pNext = hwctx->create_pnext,
2583  };
2584 
2585 #ifdef _WIN32
2586  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY)
2587  try_export_flags(hwfc, &eiinfo.handleTypes, &e, IsWindows8OrGreater()
2588  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
2589  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT);
2590 #else
2591  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY)
2592  try_export_flags(hwfc, &eiinfo.handleTypes, &e,
2593  VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT);
2594 #endif
2595 
2596  for (int i = 0; i < av_pix_fmt_count_planes(hwfc->sw_format); i++) {
2597  eminfo[i].sType = VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO;
2598  eminfo[i].pNext = hwctx->alloc_pnext[i];
2599  eminfo[i].handleTypes = e;
2600  }
2601 
2602  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2603  hwctx->nb_layers,
2604  eiinfo.handleTypes ? &eiinfo : hwctx->create_pnext);
2605  if (err)
2606  return NULL;
2607 
2608  err = alloc_bind_mem(hwfc, f, eminfo, sizeof(*eminfo));
2609  if (err)
2610  goto fail;
2611 
2612  if ( (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2613  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR))
2614  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DPB);
2615  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)
2616  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_DECODING_DST);
2617  else if (hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR)
2618  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_ENCODING_DPB);
2619  else if (hwctx->usage & VK_IMAGE_USAGE_TRANSFER_DST_BIT)
2620  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_WRITE);
2621  else
2622  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_GENERAL);
2623  if (err)
2624  goto fail;
2625 
2626  avbuf = av_buffer_create((uint8_t *)f, sizeof(AVVkFrame),
2627  vulkan_frame_free_cb, hwfc, 0);
2628  if (!avbuf)
2629  goto fail;
2630 
2631  return avbuf;
2632 
2633 fail:
2634  vulkan_frame_free(hwfc, f);
2635  return NULL;
2636 }
2637 
2638 static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2639 {
2640  pthread_mutex_lock(&vkf->internal->update_mutex);
2641 }
2642 
2643 static void unlock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
2644 {
2645  pthread_mutex_unlock(&vkf->internal->update_mutex);
2646 }
2647 
2648 static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
2649 {
2650  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2651  VulkanFramesPriv *fp = hwfc->hwctx;
2652 
2653  if (fp->modifier_info) {
2654  if (fp->modifier_info->pDrmFormatModifiers)
2655  av_freep(&fp->modifier_info->pDrmFormatModifiers);
2656  av_freep(&fp->modifier_info);
2657  }
2658 
2659  ff_vk_exec_pool_free(&p->vkctx, &fp->compute_exec);
2660  ff_vk_exec_pool_free(&p->vkctx, &fp->upload_exec);
2661  ff_vk_exec_pool_free(&p->vkctx, &fp->download_exec);
2662 
2663  av_buffer_pool_uninit(&fp->tmp);
2664 }
2665 
2666 static int vulkan_frames_init(AVHWFramesContext *hwfc)
2667 {
2668  int err;
2669  AVVkFrame *f;
2670  VulkanFramesPriv *fp = hwfc->hwctx;
2671  AVVulkanFramesContext *hwctx = &fp->p;
2672  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
2673  VkImageUsageFlagBits supported_usage;
2674  const struct FFVkFormatEntry *fmt;
2675  int disable_multiplane = p->disable_multiplane ||
2676  (hwctx->flags & AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE);
2677 
2678  /* Defaults */
2679  if (!hwctx->nb_layers)
2680  hwctx->nb_layers = 1;
2681 
2682  /* VK_IMAGE_TILING_OPTIMAL == 0, can't check for it really */
2683  if (p->use_linear_images &&
2684  (hwctx->tiling != VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT))
2685  hwctx->tiling = VK_IMAGE_TILING_LINEAR;
2686 
2687 
2688  fmt = vk_find_format_entry(hwfc->sw_format);
2689  if (!fmt) {
2690  av_log(hwfc, AV_LOG_ERROR, "Unsupported pixel format: %s!\n",
2691  av_get_pix_fmt_name(hwfc->sw_format));
2692  return AVERROR(EINVAL);
2693  }
2694 
2695  if (hwctx->format[0] != VK_FORMAT_UNDEFINED) {
2696  if (hwctx->format[0] != fmt->vkf) {
2697  for (int i = 0; i < fmt->nb_images_fallback; i++) {
2698  if (hwctx->format[i] != fmt->fallback[i]) {
2699  av_log(hwfc, AV_LOG_ERROR, "Incompatible Vulkan format given "
2700  "for the current sw_format %s!\n",
2701  av_get_pix_fmt_name(hwfc->sw_format));
2702  return AVERROR(EINVAL);
2703  }
2704  }
2705  }
2706 
2707  /* Check if the sw_format itself is supported */
2708  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
2709  hwctx->tiling, NULL,
2710  NULL, NULL, &supported_usage, 0,
2711  !hwctx->usage ||
2712  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
2713  if (err < 0) {
2714  av_log(hwfc, AV_LOG_ERROR, "Unsupported sw format: %s!\n",
2715  av_get_pix_fmt_name(hwfc->sw_format));
2716  return AVERROR(EINVAL);
2717  }
2718  } else {
2719  err = vkfmt_from_pixfmt2(hwfc->device_ctx, hwfc->sw_format,
2720  hwctx->tiling, hwctx->format, NULL,
2721  NULL, &supported_usage,
2722  disable_multiplane,
2723  !hwctx->usage ||
2724  (hwctx->usage & VK_IMAGE_USAGE_STORAGE_BIT));
2725  if (err < 0)
2726  return err;
2727  }
2728 
2729  /* Image usage flags */
2730  if (!hwctx->usage) {
2731  hwctx->usage = supported_usage & (VK_BUFFER_USAGE_TRANSFER_DST_BIT |
2732  VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
2733  VK_IMAGE_USAGE_STORAGE_BIT |
2734  VK_IMAGE_USAGE_SAMPLED_BIT);
2735 
2736  /* Enables encoding of images, if supported by format and extensions */
2737  if ((supported_usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
2738  (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
2739  FF_VK_EXT_VIDEO_MAINTENANCE_1)))
2740  hwctx->usage |= VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR;
2741  }
2742 
2743  /* Image creation flags.
2744  * Only fill them in automatically if the image is not going to be used as
2745  * a DPB-only image, and we have SAMPLED/STORAGE bits set. */
2746  if (!hwctx->img_flags) {
2747  int is_lone_dpb = ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_DPB_BIT_KHR) ||
2748  ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DPB_BIT_KHR) &&
2749  !(hwctx->usage & VK_IMAGE_USAGE_VIDEO_DECODE_DST_BIT_KHR)));
2750  int sampleable = hwctx->usage & (VK_IMAGE_USAGE_SAMPLED_BIT |
2751  VK_IMAGE_USAGE_STORAGE_BIT);
2752  hwctx->img_flags = VK_IMAGE_CREATE_MUTABLE_FORMAT_BIT;
2753  if (sampleable && !is_lone_dpb) {
2754  hwctx->img_flags |= VK_IMAGE_CREATE_ALIAS_BIT;
2755  if ((fmt->vk_planes > 1) && (hwctx->format[0] == fmt->vkf))
2756  hwctx->img_flags |= VK_IMAGE_CREATE_EXTENDED_USAGE_BIT;
2757  }
2758  }
2759 
2760  /* If the image has an ENCODE_SRC usage, and the maintenance1
2761  * extension is supported, check if it has a profile list.
2762  * If there's no profile list, or it has no encode operations,
2763  * then allow creating the image with no specific profile. */
2764  if ((hwctx->usage & VK_IMAGE_USAGE_VIDEO_ENCODE_SRC_BIT_KHR) &&
2765  (p->vkctx.extensions & (FF_VK_EXT_VIDEO_ENCODE_QUEUE |
2766  FF_VK_EXT_VIDEO_MAINTENANCE_1))) {
2767  const VkVideoProfileListInfoKHR *pl;
2768  pl = ff_vk_find_struct(hwctx->create_pnext, VK_STRUCTURE_TYPE_VIDEO_PROFILE_LIST_INFO_KHR);
2769  if (!pl) {
2770  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
2771  } else {
2772  uint32_t i;
2773  for (i = 0; i < pl->profileCount; i++) {
2774  /* Video ops start at exactly 0x00010000 */
2775  if (pl->pProfiles[i].videoCodecOperation & 0xFFFF0000)
2776  break;
2777  }
2778  if (i == pl->profileCount)
2779  hwctx->img_flags |= VK_IMAGE_CREATE_VIDEO_PROFILE_INDEPENDENT_BIT_KHR;
2780  }
2781  }
2782 
2783  if (!hwctx->lock_frame)
2784  hwctx->lock_frame = lock_frame;
2785 
2786  if (!hwctx->unlock_frame)
2787  hwctx->unlock_frame = unlock_frame;
2788 
2789  err = ff_vk_exec_pool_init(&p->vkctx, &p->compute_qf, &fp->compute_exec,
2790  p->compute_qf.nb_queues, 0, 0, 0, NULL);
2791  if (err)
2792  return err;
2793 
2794  err = ff_vk_exec_pool_init(&p->vkctx, &p->transfer_qf, &fp->upload_exec,
2795  p->transfer_qf.nb_queues*2, 0, 0, 0, NULL);
2796  if (err)
2797  return err;
2798 
2799  err = ff_vk_exec_pool_init(&p->vkctx, &p->transfer_qf, &fp->download_exec,
2800  p->transfer_qf.nb_queues, 0, 0, 0, NULL);
2801  if (err)
2802  return err;
2803 
2804  /* Test to see if allocation will fail */
2805  err = create_frame(hwfc, &f, hwctx->tiling, hwctx->usage, hwctx->img_flags,
2806  hwctx->nb_layers, hwctx->create_pnext);
2807  if (err)
2808  return err;
2809 
2810  vulkan_frame_free(hwfc, f);
2811 
2812  /* If user did not specify a pool, hwfc->pool will be set to the internal one
2813  * in hwcontext.c just after this gets called */
2814  if (!hwfc->pool) {
2815  ffhwframesctx(hwfc)->pool_internal = av_buffer_pool_init2(sizeof(AVVkFrame),
2816  hwfc, vulkan_pool_alloc,
2817  NULL);
2818  if (!ffhwframesctx(hwfc)->pool_internal)
2819  return AVERROR(ENOMEM);
2820  }
2821 
2822  return 0;
2823 }
2824 
2825 static int vulkan_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame)
2826 {
2827  frame->buf[0] = av_buffer_pool_get(hwfc->pool);
2828  if (!frame->buf[0])
2829  return AVERROR(ENOMEM);
2830 
2831  frame->data[0] = frame->buf[0]->data;
2832  frame->format = AV_PIX_FMT_VULKAN;
2833  frame->width = hwfc->width;
2834  frame->height = hwfc->height;
2835 
2836  return 0;
2837 }
2838 
2839 static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc,
2840  enum AVHWFrameTransferDirection dir,
2841  enum AVPixelFormat **formats)
2842 {
2843  enum AVPixelFormat *fmts;
2844  int n = 2;
2845 
2846 #if CONFIG_CUDA
2847  n++;
2848 #endif
2849  fmts = av_malloc_array(n, sizeof(*fmts));
2850  if (!fmts)
2851  return AVERROR(ENOMEM);
2852 
2853  n = 0;
2854  fmts[n++] = hwfc->sw_format;
2855 #if CONFIG_CUDA
2856  fmts[n++] = AV_PIX_FMT_CUDA;
2857 #endif
2858  fmts[n++] = AV_PIX_FMT_NONE;
2859 
2860  *formats = fmts;
2861  return 0;
2862 }
2863 
2864 #if CONFIG_LIBDRM
2865 static void vulkan_unmap_from_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
2866 {
2867  vulkan_frame_free(hwfc, hwmap->priv);
2868 }
2869 
2870 static const struct {
2871  uint32_t drm_fourcc;
2872  VkFormat vk_format;
2873 } vulkan_drm_format_map[] = {
2874  { DRM_FORMAT_R8, VK_FORMAT_R8_UNORM },
2875  { DRM_FORMAT_R16, VK_FORMAT_R16_UNORM },
2876  { DRM_FORMAT_GR88, VK_FORMAT_R8G8_UNORM },
2877  { DRM_FORMAT_RG88, VK_FORMAT_R8G8_UNORM },
2878  { DRM_FORMAT_GR1616, VK_FORMAT_R16G16_UNORM },
2879  { DRM_FORMAT_RG1616, VK_FORMAT_R16G16_UNORM },
2880  { DRM_FORMAT_ARGB8888, VK_FORMAT_B8G8R8A8_UNORM },
2881  { DRM_FORMAT_XRGB8888, VK_FORMAT_B8G8R8A8_UNORM },
2882  { DRM_FORMAT_ABGR8888, VK_FORMAT_R8G8B8A8_UNORM },
2883  { DRM_FORMAT_XBGR8888, VK_FORMAT_R8G8B8A8_UNORM },
2884  { DRM_FORMAT_ARGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
2885  { DRM_FORMAT_ABGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
2886  { DRM_FORMAT_XRGB2101010, VK_FORMAT_A2B10G10R10_UNORM_PACK32 },
2887  { DRM_FORMAT_XBGR2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 },
2888 
2889  // All these DRM_FORMATs were added in the same libdrm commit.
2890 #ifdef DRM_FORMAT_XYUV8888
2891  { DRM_FORMAT_XYUV8888, VK_FORMAT_R8G8B8A8_UNORM },
2892  { DRM_FORMAT_XVYU2101010, VK_FORMAT_A2R10G10B10_UNORM_PACK32 } ,
2893  { DRM_FORMAT_XVYU12_16161616, VK_FORMAT_R12X4G12X4B12X4A12X4_UNORM_4PACK16 } ,
2894  { DRM_FORMAT_XVYU16161616, VK_FORMAT_R16G16B16A16_UNORM } ,
2895 #endif
2896 };
2897 
2898 static inline VkFormat drm_to_vulkan_fmt(uint32_t drm_fourcc)
2899 {
2900  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
2901  if (vulkan_drm_format_map[i].drm_fourcc == drm_fourcc)
2902  return vulkan_drm_format_map[i].vk_format;
2903  return VK_FORMAT_UNDEFINED;
2904 }
2905 
2906 static int vulkan_map_from_drm_frame_desc(AVHWFramesContext *hwfc, AVVkFrame **frame,
2907  const AVFrame *src, int flags)
2908 {
2909  int err = 0;
2910  VkResult ret;
2911  AVVkFrame *f;
2912  int bind_counts = 0;
2913  AVHWDeviceContext *ctx = hwfc->device_ctx;
2914  VulkanDevicePriv *p = ctx->hwctx;
2915  AVVulkanDeviceContext *hwctx = &p->p;
2916  FFVulkanFunctions *vk = &p->vkctx.vkfn;
2917  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
2918  VkBindImageMemoryInfo bind_info[AV_DRM_MAX_PLANES];
2919  VkBindImagePlaneMemoryInfo plane_info[AV_DRM_MAX_PLANES];
2920 
2921  for (int i = 0; i < desc->nb_layers; i++) {
2922  if (drm_to_vulkan_fmt(desc->layers[i].format) == VK_FORMAT_UNDEFINED) {
2923  av_log(ctx, AV_LOG_ERROR, "Unsupported DMABUF layer format %#08x!\n",
2924  desc->layers[i].format);
2925  return AVERROR(EINVAL);
2926  }
2927  }
2928 
2929  if (!(f = av_vk_frame_alloc())) {
2930  av_log(ctx, AV_LOG_ERROR, "Unable to allocate memory for AVVkFrame!\n");
2931  err = AVERROR(ENOMEM);
2932  goto fail;
2933  }
2934 
2935  f->tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT;
2936 
2937  for (int i = 0; i < desc->nb_layers; i++) {
2938  const int planes = desc->layers[i].nb_planes;
2939 
2940  /* Semaphore */
2941  VkSemaphoreTypeCreateInfo sem_type_info = {
2942  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
2943  .semaphoreType = VK_SEMAPHORE_TYPE_TIMELINE,
2944  .initialValue = 0,
2945  };
2946  VkSemaphoreCreateInfo sem_spawn = {
2947  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
2948  .pNext = &sem_type_info,
2949  };
2950 
2951  /* Image creation */
2952  VkSubresourceLayout ext_img_layouts[AV_DRM_MAX_PLANES];
2953  VkImageDrmFormatModifierExplicitCreateInfoEXT ext_img_mod_spec = {
2954  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_EXPLICIT_CREATE_INFO_EXT,
2955  .drmFormatModifier = desc->objects[0].format_modifier,
2956  .drmFormatModifierPlaneCount = planes,
2957  .pPlaneLayouts = (const VkSubresourceLayout *)&ext_img_layouts,
2958  };
2959  VkExternalMemoryImageCreateInfo ext_img_spec = {
2960  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMAGE_CREATE_INFO,
2961  .pNext = &ext_img_mod_spec,
2962  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
2963  };
2964  VkImageCreateInfo create_info = {
2965  .sType = VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO,
2966  .pNext = &ext_img_spec,
2967  .imageType = VK_IMAGE_TYPE_2D,
2968  .format = drm_to_vulkan_fmt(desc->layers[i].format),
2969  .extent.depth = 1,
2970  .mipLevels = 1,
2971  .arrayLayers = 1,
2972  .flags = 0x0,
2973  .tiling = VK_IMAGE_TILING_DRM_FORMAT_MODIFIER_EXT,
2974  .initialLayout = VK_IMAGE_LAYOUT_UNDEFINED, /* specs say so */
2975  .usage = 0x0, /* filled in below */
2976  .samples = VK_SAMPLE_COUNT_1_BIT,
2977  .pQueueFamilyIndices = p->img_qfs,
2978  .queueFamilyIndexCount = p->nb_img_qfs,
2979  .sharingMode = p->nb_img_qfs > 1 ? VK_SHARING_MODE_CONCURRENT :
2980  VK_SHARING_MODE_EXCLUSIVE,
2981  };
2982 
2983  /* Image format verification */
2984  VkExternalImageFormatProperties ext_props = {
2985  .sType = VK_STRUCTURE_TYPE_EXTERNAL_IMAGE_FORMAT_PROPERTIES_KHR,
2986  };
2987  VkImageFormatProperties2 props_ret = {
2988  .sType = VK_STRUCTURE_TYPE_IMAGE_FORMAT_PROPERTIES_2,
2989  .pNext = &ext_props,
2990  };
2991  VkPhysicalDeviceImageDrmFormatModifierInfoEXT props_drm_mod = {
2992  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_DRM_FORMAT_MODIFIER_INFO_EXT,
2993  .drmFormatModifier = ext_img_mod_spec.drmFormatModifier,
2994  .pQueueFamilyIndices = create_info.pQueueFamilyIndices,
2995  .queueFamilyIndexCount = create_info.queueFamilyIndexCount,
2996  .sharingMode = create_info.sharingMode,
2997  };
2998  VkPhysicalDeviceExternalImageFormatInfo props_ext = {
2999  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_EXTERNAL_IMAGE_FORMAT_INFO,
3000  .pNext = &props_drm_mod,
3001  .handleType = ext_img_spec.handleTypes,
3002  };
3003  VkPhysicalDeviceImageFormatInfo2 fmt_props;
3004 
3005  if (flags & AV_HWFRAME_MAP_READ)
3006  create_info.usage |= VK_IMAGE_USAGE_SAMPLED_BIT |
3007  VK_IMAGE_USAGE_TRANSFER_SRC_BIT;
3008  if (flags & AV_HWFRAME_MAP_WRITE)
3009  create_info.usage |= VK_IMAGE_USAGE_STORAGE_BIT |
3010  VK_IMAGE_USAGE_TRANSFER_DST_BIT;
3011 
3012  fmt_props = (VkPhysicalDeviceImageFormatInfo2) {
3013  .sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_IMAGE_FORMAT_INFO_2,
3014  .pNext = &props_ext,
3015  .format = create_info.format,
3016  .type = create_info.imageType,
3017  .tiling = create_info.tiling,
3018  .usage = create_info.usage,
3019  .flags = create_info.flags,
3020  };
3021 
3022  /* Check if importing is possible for this combination of parameters */
3023  ret = vk->GetPhysicalDeviceImageFormatProperties2(hwctx->phys_dev,
3024  &fmt_props, &props_ret);
3025  if (ret != VK_SUCCESS) {
3026  av_log(ctx, AV_LOG_ERROR, "Cannot map DRM frame to Vulkan: %s\n",
3027  ff_vk_ret2str(ret));
3028  err = AVERROR_EXTERNAL;
3029  goto fail;
3030  }
3031 
3032  /* Set the image width/height */
3033  get_plane_wh(&create_info.extent.width, &create_info.extent.height,
3034  hwfc->sw_format, src->width, src->height, i);
3035 
3036  /* Set the subresource layout based on the layer properties */
3037  for (int j = 0; j < planes; j++) {
3038  ext_img_layouts[j].offset = desc->layers[i].planes[j].offset;
3039  ext_img_layouts[j].rowPitch = desc->layers[i].planes[j].pitch;
3040  ext_img_layouts[j].size = 0; /* The specs say so for all 3 */
3041  ext_img_layouts[j].arrayPitch = 0;
3042  ext_img_layouts[j].depthPitch = 0;
3043  }
3044 
3045  /* Create image */
3046  ret = vk->CreateImage(hwctx->act_dev, &create_info,
3047  hwctx->alloc, &f->img[i]);
3048  if (ret != VK_SUCCESS) {
3049  av_log(ctx, AV_LOG_ERROR, "Image creation failure: %s\n",
3050  ff_vk_ret2str(ret));
3051  err = AVERROR(EINVAL);
3052  goto fail;
3053  }
3054 
3055  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3056  hwctx->alloc, &f->sem[i]);
3057  if (ret != VK_SUCCESS) {
3058  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3059  ff_vk_ret2str(ret));
3060  err = AVERROR_EXTERNAL;
3061  goto fail;
3062  }
3063 
3064  f->queue_family[i] = VK_QUEUE_FAMILY_EXTERNAL;
3065  f->layout[i] = create_info.initialLayout;
3066  f->access[i] = 0x0;
3067  f->sem_value[i] = 0;
3068  }
3069 
3070  for (int i = 0; i < desc->nb_layers; i++) {
3071  /* Memory requirements */
3072  VkImageMemoryRequirementsInfo2 req_desc = {
3073  .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2,
3074  .image = f->img[i],
3075  };
3076  VkMemoryDedicatedRequirements ded_req = {
3077  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS,
3078  };
3079  VkMemoryRequirements2 req2 = {
3080  .sType = VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2,
3081  .pNext = &ded_req,
3082  };
3083 
3084  /* Allocation/importing */
3085  VkMemoryFdPropertiesKHR fdmp = {
3086  .sType = VK_STRUCTURE_TYPE_MEMORY_FD_PROPERTIES_KHR,
3087  };
3088  /* This assumes that a layer will never be constructed from multiple
3089  * objects. If that was to happen in the real world, this code would
3090  * need to import each plane separately.
3091  */
3092  VkImportMemoryFdInfoKHR idesc = {
3093  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_FD_INFO_KHR,
3094  .fd = dup(desc->objects[desc->layers[i].planes[0].object_index].fd),
3095  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3096  };
3097  VkMemoryDedicatedAllocateInfo ded_alloc = {
3098  .sType = VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO,
3099  .pNext = &idesc,
3100  .image = req_desc.image,
3101  };
3102 
3103  /* Get object properties */
3104  ret = vk->GetMemoryFdPropertiesKHR(hwctx->act_dev,
3105  VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3106  idesc.fd, &fdmp);
3107  if (ret != VK_SUCCESS) {
3108  av_log(hwfc, AV_LOG_ERROR, "Failed to get FD properties: %s\n",
3109  ff_vk_ret2str(ret));
3110  err = AVERROR_EXTERNAL;
3111  close(idesc.fd);
3112  goto fail;
3113  }
3114 
3115  vk->GetImageMemoryRequirements2(hwctx->act_dev, &req_desc, &req2);
3116 
3117  /* Only a single bit must be set, not a range, and it must match */
3118  req2.memoryRequirements.memoryTypeBits = fdmp.memoryTypeBits;
3119 
3120  err = alloc_mem(ctx, &req2.memoryRequirements,
3121  VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
3122  (ded_req.prefersDedicatedAllocation ||
3123  ded_req.requiresDedicatedAllocation) ?
3124  &ded_alloc : ded_alloc.pNext,
3125  &f->flags, &f->mem[i]);
3126  if (err) {
3127  close(idesc.fd);
3128  return err;
3129  }
3130 
3131  f->size[i] = req2.memoryRequirements.size;
3132  }
3133 
3134  for (int i = 0; i < desc->nb_layers; i++) {
3135  const int planes = desc->layers[i].nb_planes;
3136  for (int j = 0; j < planes; j++) {
3137  VkImageAspectFlagBits aspect = j == 0 ? VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT :
3138  j == 1 ? VK_IMAGE_ASPECT_MEMORY_PLANE_1_BIT_EXT :
3139  VK_IMAGE_ASPECT_MEMORY_PLANE_2_BIT_EXT;
3140 
3141  plane_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_PLANE_MEMORY_INFO;
3142  plane_info[bind_counts].pNext = NULL;
3143  plane_info[bind_counts].planeAspect = aspect;
3144 
3145  bind_info[bind_counts].sType = VK_STRUCTURE_TYPE_BIND_IMAGE_MEMORY_INFO;
3146  bind_info[bind_counts].pNext = planes > 1 ? &plane_info[bind_counts] : NULL;
3147  bind_info[bind_counts].image = f->img[i];
3148  bind_info[bind_counts].memory = f->mem[i];
3149 
3150  /* Offset is already signalled via pPlaneLayouts above */
3151  bind_info[bind_counts].memoryOffset = 0;
3152 
3153  bind_counts++;
3154  }
3155  }
3156 
3157  /* Bind the allocated memory to the images */
3158  ret = vk->BindImageMemory2(hwctx->act_dev, bind_counts, bind_info);
3159  if (ret != VK_SUCCESS) {
3160  av_log(ctx, AV_LOG_ERROR, "Failed to bind memory: %s\n",
3161  ff_vk_ret2str(ret));
3162  err = AVERROR_EXTERNAL;
3163  goto fail;
3164  }
3165 
3166  *frame = f;
3167 
3168  return 0;
3169 
3170 fail:
3171  vulkan_frame_free(hwfc, f);
3172 
3173  return err;
3174 }
3175 
3176 static int vulkan_map_from_drm_frame_sync(AVHWFramesContext *hwfc, AVFrame *dst,
3177  const AVFrame *src, int flags)
3178 {
3179  int err;
3180  VkResult ret;
3181  AVHWDeviceContext *ctx = hwfc->device_ctx;
3182  VulkanDevicePriv *p = ctx->hwctx;
3183  VulkanFramesPriv *fp = hwfc->hwctx;
3184  AVVulkanDeviceContext *hwctx = &p->p;
3185  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3186 
3187  const AVDRMFrameDescriptor *desc = (AVDRMFrameDescriptor *)src->data[0];
3188 
3189 #ifdef DMA_BUF_IOCTL_EXPORT_SYNC_FILE
3190  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM) {
3191  VkCommandBuffer cmd_buf;
3192  FFVkExecContext *exec;
3193  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
3194  VkSemaphore drm_sync_sem[AV_DRM_MAX_PLANES] = { 0 };
3195  int nb_img_bar = 0;
3196 
3197  for (int i = 0; i < desc->nb_objects; i++) {
3198  VkSemaphoreTypeCreateInfo sem_type_info = {
3199  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_TYPE_CREATE_INFO,
3200  .semaphoreType = VK_SEMAPHORE_TYPE_BINARY,
3201  };
3202  VkSemaphoreCreateInfo sem_spawn = {
3203  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
3204  .pNext = &sem_type_info,
3205  };
3206  VkImportSemaphoreFdInfoKHR import_info;
3207  struct dma_buf_export_sync_file implicit_fd_info = {
3208  .flags = DMA_BUF_SYNC_READ,
3209  .fd = -1,
3210  };
3211 
3212  if (ioctl(desc->objects[i].fd, DMA_BUF_IOCTL_EXPORT_SYNC_FILE,
3213  &implicit_fd_info)) {
3214  err = AVERROR(errno);
3215  av_log(hwctx, AV_LOG_ERROR, "Failed to retrieve implicit DRM sync file: %s\n",
3216  av_err2str(err));
3217  for (; i >= 0; i--)
3218  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3219  return err;
3220  }
3221 
3222  ret = vk->CreateSemaphore(hwctx->act_dev, &sem_spawn,
3223  hwctx->alloc, &drm_sync_sem[i]);
3224  if (ret != VK_SUCCESS) {
3225  av_log(hwctx, AV_LOG_ERROR, "Failed to create semaphore: %s\n",
3226  ff_vk_ret2str(ret));
3227  err = AVERROR_EXTERNAL;
3228  for (; i >= 0; i--)
3229  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3230  return err;
3231  }
3232 
3233  import_info = (VkImportSemaphoreFdInfoKHR) {
3234  .sType = VK_STRUCTURE_TYPE_IMPORT_SEMAPHORE_FD_INFO_KHR,
3235  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT,
3236  .flags = VK_SEMAPHORE_IMPORT_TEMPORARY_BIT,
3237  .semaphore = drm_sync_sem[i],
3238  .fd = implicit_fd_info.fd,
3239  };
3240 
3241  ret = vk->ImportSemaphoreFdKHR(hwctx->act_dev, &import_info);
3242  if (ret != VK_SUCCESS) {
3243  av_log(hwctx, AV_LOG_ERROR, "Failed to import semaphore: %s\n",
3244  ff_vk_ret2str(ret));
3245  err = AVERROR_EXTERNAL;
3246  for (; i >= 0; i--)
3247  vk->DestroySemaphore(hwctx->act_dev, drm_sync_sem[i], hwctx->alloc);
3248  return err;
3249  }
3250  }
3251 
3252  exec = ff_vk_exec_get(&p->vkctx, &fp->compute_exec);
3253  cmd_buf = exec->buf;
3254 
3255  ff_vk_exec_start(&p->vkctx, exec);
3256 
3257  /* Ownership of semaphores is passed */
3258  err = ff_vk_exec_add_dep_bool_sem(&p->vkctx, exec,
3259  drm_sync_sem, desc->nb_objects,
3260  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT, 1);
3261  if (err < 0)
3262  return err;
3263 
3264  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, dst,
3265  VK_PIPELINE_STAGE_2_NONE,
3266  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT);
3267  if (err < 0)
3268  return err;
3269 
3270  ff_vk_frame_barrier(&p->vkctx, exec, dst, img_bar, &nb_img_bar,
3271  VK_PIPELINE_STAGE_2_NONE,
3272  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
3273  ((flags & AV_HWFRAME_MAP_READ) ?
3274  VK_ACCESS_2_SHADER_SAMPLED_READ_BIT : 0x0) |
3275  ((flags & AV_HWFRAME_MAP_WRITE) ?
3276  VK_ACCESS_2_SHADER_STORAGE_WRITE_BIT : 0x0),
3277  VK_IMAGE_LAYOUT_GENERAL,
3278  VK_QUEUE_FAMILY_IGNORED);
3279 
3280  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
3281  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
3282  .pImageMemoryBarriers = img_bar,
3283  .imageMemoryBarrierCount = nb_img_bar,
3284  });
3285 
3286  err = ff_vk_exec_submit(&p->vkctx, exec);
3287  if (err < 0)
3288  return err;
3289  } else
3290 #endif
3291  {
3292  AVVkFrame *f = (AVVkFrame *)dst->data[0];
3293  av_log(hwctx, AV_LOG_WARNING, "No support for synchronization when importing DMA-BUFs, "
3294  "image may be corrupted.\n");
3295  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_IMPORT);
3296  if (err)
3297  return err;
3298  }
3299 
3300  return 0;
3301 }
3302 
3303 static int vulkan_map_from_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3304  const AVFrame *src, int flags)
3305 {
3306  int err = 0;
3307  AVVkFrame *f;
3308 
3309  if ((err = vulkan_map_from_drm_frame_desc(hwfc, &f, src, flags)))
3310  return err;
3311 
3312  /* The unmapping function will free this */
3313  dst->data[0] = (uint8_t *)f;
3314  dst->width = src->width;
3315  dst->height = src->height;
3316 
3317  err = ff_hwframe_map_create(dst->hw_frames_ctx, dst, src,
3318  &vulkan_unmap_from_drm, f);
3319  if (err < 0)
3320  goto fail;
3321 
3322  err = vulkan_map_from_drm_frame_sync(hwfc, dst, src, flags);
3323  if (err < 0)
3324  return err;
3325 
3326  av_log(hwfc, AV_LOG_DEBUG, "Mapped DRM object to Vulkan!\n");
3327 
3328  return 0;
3329 
3330 fail:
3331  vulkan_frame_free(hwfc->device_ctx->hwctx, f);
3332  dst->data[0] = NULL;
3333  return err;
3334 }
3335 
3336 #if CONFIG_VAAPI
3337 static int vulkan_map_from_vaapi(AVHWFramesContext *dst_fc,
3338  AVFrame *dst, const AVFrame *src,
3339  int flags)
3340 {
3341  int err;
3342  AVFrame *tmp = av_frame_alloc();
3343  AVHWFramesContext *vaapi_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3344  AVVAAPIDeviceContext *vaapi_ctx = vaapi_fc->device_ctx->hwctx;
3345  VASurfaceID surface_id = (VASurfaceID)(uintptr_t)src->data[3];
3346 
3347  if (!tmp)
3348  return AVERROR(ENOMEM);
3349 
3350  /* We have to sync since like the previous comment said, no semaphores */
3351  vaSyncSurface(vaapi_ctx->display, surface_id);
3352 
3353  tmp->format = AV_PIX_FMT_DRM_PRIME;
3354 
3355  err = av_hwframe_map(tmp, src, flags);
3356  if (err < 0)
3357  goto fail;
3358 
3359  err = vulkan_map_from_drm(dst_fc, dst, tmp, flags);
3360  if (err < 0)
3361  goto fail;
3362 
3363  err = ff_hwframe_map_replace(dst, src);
3364 
3365 fail:
3366  av_frame_free(&tmp);
3367  return err;
3368 }
3369 #endif
3370 #endif
3371 
3372 #if CONFIG_CUDA
3373 static int vulkan_export_to_cuda(AVHWFramesContext *hwfc,
3374  AVBufferRef *cuda_hwfc,
3375  const AVFrame *frame)
3376 {
3377  int err;
3378  VkResult ret;
3379  AVVkFrame *dst_f;
3380  AVVkFrameInternal *dst_int;
3381  AVHWDeviceContext *ctx = hwfc->device_ctx;
3382  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3383  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3384  VulkanDevicePriv *p = ctx->hwctx;
3385  AVVulkanDeviceContext *hwctx = &p->p;
3386  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3387 
3388  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)cuda_hwfc->data;
3389  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3390  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3391  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3392  CudaFunctions *cu = cu_internal->cuda_dl;
3393  CUarray_format cufmt = desc->comp[0].depth > 8 ? CU_AD_FORMAT_UNSIGNED_INT16 :
3394  CU_AD_FORMAT_UNSIGNED_INT8;
3395 
3396  dst_f = (AVVkFrame *)frame->data[0];
3397  dst_int = dst_f->internal;
3398 
3399  if (!dst_int->cuda_fc_ref) {
3400  dst_int->cuda_fc_ref = av_buffer_ref(cuda_hwfc);
3401  if (!dst_int->cuda_fc_ref)
3402  return AVERROR(ENOMEM);
3403 
3404  for (int i = 0; i < planes; i++) {
3405  CUDA_EXTERNAL_MEMORY_MIPMAPPED_ARRAY_DESC tex_desc = {
3406  .offset = 0,
3407  .arrayDesc = {
3408  .Depth = 0,
3409  .Format = cufmt,
3410  .NumChannels = 1 + ((planes == 2) && i),
3411  .Flags = 0,
3412  },
3413  .numLevels = 1,
3414  };
3415  int p_w, p_h;
3416 
3417 #ifdef _WIN32
3418  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3419  .type = IsWindows8OrGreater()
3420  ? CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32
3421  : CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT,
3422  .size = dst_f->size[i],
3423  };
3424  VkMemoryGetWin32HandleInfoKHR export_info = {
3425  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_WIN32_HANDLE_INFO_KHR,
3426  .memory = dst_f->mem[i],
3427  .handleType = IsWindows8OrGreater()
3428  ? VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_BIT
3429  : VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3430  };
3431  VkSemaphoreGetWin32HandleInfoKHR sem_export = {
3432  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_WIN32_HANDLE_INFO_KHR,
3433  .semaphore = dst_f->sem[i],
3434  .handleType = IsWindows8OrGreater()
3435  ? VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_BIT
3436  : VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_WIN32_KMT_BIT,
3437  };
3438  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3439  .type = 10 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_WIN32 */,
3440  };
3441 
3442  ret = vk->GetMemoryWin32HandleKHR(hwctx->act_dev, &export_info,
3443  &ext_desc.handle.win32.handle);
3444  if (ret != VK_SUCCESS) {
3445  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a Win32 Handle: %s!\n",
3446  ff_vk_ret2str(ret));
3447  err = AVERROR_EXTERNAL;
3448  goto fail;
3449  }
3450  dst_int->ext_mem_handle[i] = ext_desc.handle.win32.handle;
3451 #else
3452  CUDA_EXTERNAL_MEMORY_HANDLE_DESC ext_desc = {
3453  .type = CU_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD,
3454  .size = dst_f->size[i],
3455  };
3456  VkMemoryGetFdInfoKHR export_info = {
3457  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3458  .memory = dst_f->mem[i],
3459  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_OPAQUE_FD_BIT_KHR,
3460  };
3461  VkSemaphoreGetFdInfoKHR sem_export = {
3462  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_GET_FD_INFO_KHR,
3463  .semaphore = dst_f->sem[i],
3464  .handleType = VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_OPAQUE_FD_BIT,
3465  };
3466  CUDA_EXTERNAL_SEMAPHORE_HANDLE_DESC ext_sem_desc = {
3467  .type = 9 /* TODO: CU_EXTERNAL_SEMAPHORE_HANDLE_TYPE_TIMELINE_SEMAPHORE_FD */,
3468  };
3469 
3470  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3471  &ext_desc.handle.fd);
3472  if (ret != VK_SUCCESS) {
3473  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD: %s!\n",
3474  ff_vk_ret2str(ret));
3475  err = AVERROR_EXTERNAL;
3476  goto fail;
3477  }
3478 #endif
3479 
3480  ret = CHECK_CU(cu->cuImportExternalMemory(&dst_int->ext_mem[i], &ext_desc));
3481  if (ret < 0) {
3482 #ifndef _WIN32
3483  close(ext_desc.handle.fd);
3484 #endif
3485  err = AVERROR_EXTERNAL;
3486  goto fail;
3487  }
3488 
3489  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3490  tex_desc.arrayDesc.Width = p_w;
3491  tex_desc.arrayDesc.Height = p_h;
3492 
3493  ret = CHECK_CU(cu->cuExternalMemoryGetMappedMipmappedArray(&dst_int->cu_mma[i],
3494  dst_int->ext_mem[i],
3495  &tex_desc));
3496  if (ret < 0) {
3497  err = AVERROR_EXTERNAL;
3498  goto fail;
3499  }
3500 
3501  ret = CHECK_CU(cu->cuMipmappedArrayGetLevel(&dst_int->cu_array[i],
3502  dst_int->cu_mma[i], 0));
3503  if (ret < 0) {
3504  err = AVERROR_EXTERNAL;
3505  goto fail;
3506  }
3507 
3508 #ifdef _WIN32
3509  ret = vk->GetSemaphoreWin32HandleKHR(hwctx->act_dev, &sem_export,
3510  &ext_sem_desc.handle.win32.handle);
3511 #else
3512  ret = vk->GetSemaphoreFdKHR(hwctx->act_dev, &sem_export,
3513  &ext_sem_desc.handle.fd);
3514 #endif
3515  if (ret != VK_SUCCESS) {
3516  av_log(ctx, AV_LOG_ERROR, "Failed to export semaphore: %s\n",
3517  ff_vk_ret2str(ret));
3518  err = AVERROR_EXTERNAL;
3519  goto fail;
3520  }
3521 #ifdef _WIN32
3522  dst_int->ext_sem_handle[i] = ext_sem_desc.handle.win32.handle;
3523 #endif
3524 
3525  ret = CHECK_CU(cu->cuImportExternalSemaphore(&dst_int->cu_sem[i],
3526  &ext_sem_desc));
3527  if (ret < 0) {
3528 #ifndef _WIN32
3529  close(ext_sem_desc.handle.fd);
3530 #endif
3531  err = AVERROR_EXTERNAL;
3532  goto fail;
3533  }
3534  }
3535  }
3536 
3537  return 0;
3538 
3539 fail:
3540  vulkan_free_internal(dst_f);
3541  return err;
3542 }
3543 
3544 static int vulkan_transfer_data_from_cuda(AVHWFramesContext *hwfc,
3545  AVFrame *dst, const AVFrame *src)
3546 {
3547  int err;
3548  CUcontext dummy;
3549  AVVkFrame *dst_f;
3550  AVVkFrameInternal *dst_int;
3551  VulkanFramesPriv *fp = hwfc->hwctx;
3552  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3553  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
3554 
3555  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)src->hw_frames_ctx->data;
3556  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
3557  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
3558  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
3559  CudaFunctions *cu = cu_internal->cuda_dl;
3560  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
3561  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
3562 
3563  dst_f = (AVVkFrame *)dst->data[0];
3564 
3565  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
3566  if (err < 0)
3567  return err;
3568 
3569  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
3570  if (err < 0)
3571  return err;
3572 
3573  err = vulkan_export_to_cuda(hwfc, src->hw_frames_ctx, dst);
3574  if (err < 0) {
3575  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3576  return err;
3577  }
3578 
3579  dst_int = dst_f->internal;
3580 
3581  for (int i = 0; i < planes; i++) {
3582  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
3583  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
3584  }
3585 
3586  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
3587  planes, cuda_dev->stream));
3588  if (err < 0)
3589  goto fail;
3590 
3591  for (int i = 0; i < planes; i++) {
3592  CUDA_MEMCPY2D cpy = {
3593  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
3594  .srcDevice = (CUdeviceptr)src->data[i],
3595  .srcPitch = src->linesize[i],
3596  .srcY = 0,
3597 
3598  .dstMemoryType = CU_MEMORYTYPE_ARRAY,
3599  .dstArray = dst_int->cu_array[i],
3600  };
3601 
3602  int p_w, p_h;
3603  get_plane_wh(&p_w, &p_h, hwfc->sw_format, hwfc->width, hwfc->height, i);
3604 
3605  cpy.WidthInBytes = p_w * desc->comp[i].step;
3606  cpy.Height = p_h;
3607 
3608  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
3609  if (err < 0)
3610  goto fail;
3611  }
3612 
3613  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
3614  planes, cuda_dev->stream));
3615  if (err < 0)
3616  goto fail;
3617 
3618  for (int i = 0; i < planes; i++)
3619  dst_f->sem_value[i]++;
3620 
3621  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3622 
3623  av_log(hwfc, AV_LOG_VERBOSE, "Transferred CUDA image to Vulkan!\n");
3624 
3625  return err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
3626 
3627 fail:
3628  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
3629  vulkan_free_internal(dst_f);
3630  av_buffer_unref(&dst->buf[0]);
3631  return err;
3632 }
3633 #endif
3634 
3635 static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst,
3636  const AVFrame *src, int flags)
3637 {
3638  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3639 
3640  switch (src->format) {
3641 #if CONFIG_LIBDRM
3642 #if CONFIG_VAAPI
3643  case AV_PIX_FMT_VAAPI:
3644  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3645  return vulkan_map_from_vaapi(hwfc, dst, src, flags);
3646  else
3647  return AVERROR(ENOSYS);
3648 #endif
3649  case AV_PIX_FMT_DRM_PRIME:
3650  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3651  return vulkan_map_from_drm(hwfc, dst, src, flags);
3652  else
3653  return AVERROR(ENOSYS);
3654 #endif
3655  default:
3656  return AVERROR(ENOSYS);
3657  }
3658 }
3659 
3660 #if CONFIG_LIBDRM
3661 typedef struct VulkanDRMMapping {
3662  AVDRMFrameDescriptor drm_desc;
3663  AVVkFrame *source;
3664 } VulkanDRMMapping;
3665 
3666 static void vulkan_unmap_to_drm(AVHWFramesContext *hwfc, HWMapDescriptor *hwmap)
3667 {
3668  AVDRMFrameDescriptor *drm_desc = hwmap->priv;
3669 
3670  for (int i = 0; i < drm_desc->nb_objects; i++)
3671  close(drm_desc->objects[i].fd);
3672 
3673  av_free(drm_desc);
3674 }
3675 
3676 static inline uint32_t vulkan_fmt_to_drm(VkFormat vkfmt)
3677 {
3678  for (int i = 0; i < FF_ARRAY_ELEMS(vulkan_drm_format_map); i++)
3679  if (vulkan_drm_format_map[i].vk_format == vkfmt)
3680  return vulkan_drm_format_map[i].drm_fourcc;
3681  return DRM_FORMAT_INVALID;
3682 }
3683 
3684 static int vulkan_map_to_drm(AVHWFramesContext *hwfc, AVFrame *dst,
3685  const AVFrame *src, int flags)
3686 {
3687  int err = 0;
3688  VkResult ret;
3689  AVVkFrame *f = (AVVkFrame *)src->data[0];
3690  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3691  AVVulkanDeviceContext *hwctx = &p->p;
3692  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3693  VulkanFramesPriv *fp = hwfc->hwctx;
3694  AVVulkanFramesContext *hwfctx = &fp->p;
3695  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
3696  VkImageDrmFormatModifierPropertiesEXT drm_mod = {
3697  .sType = VK_STRUCTURE_TYPE_IMAGE_DRM_FORMAT_MODIFIER_PROPERTIES_EXT,
3698  };
3699  VkSemaphoreWaitInfo wait_info = {
3700  .sType = VK_STRUCTURE_TYPE_SEMAPHORE_WAIT_INFO,
3701  .flags = 0x0,
3702  .semaphoreCount = planes,
3703  };
3704 
3705  AVDRMFrameDescriptor *drm_desc = av_mallocz(sizeof(*drm_desc));
3706  if (!drm_desc)
3707  return AVERROR(ENOMEM);
3708 
3709  err = prepare_frame(hwfc, &fp->compute_exec, f, PREP_MODE_EXTERNAL_EXPORT);
3710  if (err < 0)
3711  goto end;
3712 
3713  /* Wait for the operation to finish so we can cleanly export it. */
3714  wait_info.pSemaphores = f->sem;
3715  wait_info.pValues = f->sem_value;
3716 
3717  vk->WaitSemaphores(hwctx->act_dev, &wait_info, UINT64_MAX);
3718 
3719  err = ff_hwframe_map_create(src->hw_frames_ctx, dst, src, &vulkan_unmap_to_drm, drm_desc);
3720  if (err < 0)
3721  goto end;
3722 
3723  ret = vk->GetImageDrmFormatModifierPropertiesEXT(hwctx->act_dev, f->img[0],
3724  &drm_mod);
3725  if (ret != VK_SUCCESS) {
3726  av_log(hwfc, AV_LOG_ERROR, "Failed to retrieve DRM format modifier!\n");
3727  err = AVERROR_EXTERNAL;
3728  goto end;
3729  }
3730 
3731  for (int i = 0; (i < planes) && (f->mem[i]); i++) {
3732  VkMemoryGetFdInfoKHR export_info = {
3733  .sType = VK_STRUCTURE_TYPE_MEMORY_GET_FD_INFO_KHR,
3734  .memory = f->mem[i],
3735  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_DMA_BUF_BIT_EXT,
3736  };
3737 
3738  ret = vk->GetMemoryFdKHR(hwctx->act_dev, &export_info,
3739  &drm_desc->objects[i].fd);
3740  if (ret != VK_SUCCESS) {
3741  av_log(hwfc, AV_LOG_ERROR, "Unable to export the image as a FD!\n");
3742  err = AVERROR_EXTERNAL;
3743  goto end;
3744  }
3745 
3746  drm_desc->nb_objects++;
3747  drm_desc->objects[i].size = f->size[i];
3748  drm_desc->objects[i].format_modifier = drm_mod.drmFormatModifier;
3749  }
3750 
3751  drm_desc->nb_layers = planes;
3752  for (int i = 0; i < drm_desc->nb_layers; i++) {
3753  VkSubresourceLayout layout;
3754  VkImageSubresource sub = {
3755  .aspectMask = VK_IMAGE_ASPECT_MEMORY_PLANE_0_BIT_EXT,
3756  };
3757  VkFormat plane_vkfmt = av_vkfmt_from_pixfmt(hwfc->sw_format)[i];
3758 
3759  drm_desc->layers[i].format = vulkan_fmt_to_drm(plane_vkfmt);
3760  drm_desc->layers[i].nb_planes = 1;
3761 
3762  if (drm_desc->layers[i].format == DRM_FORMAT_INVALID) {
3763  av_log(hwfc, AV_LOG_ERROR, "Cannot map to DRM layer, unsupported!\n");
3764  err = AVERROR_PATCHWELCOME;
3765  goto end;
3766  }
3767 
3768  drm_desc->layers[i].planes[0].object_index = FFMIN(i, drm_desc->nb_objects - 1);
3769 
3770  if (f->tiling == VK_IMAGE_TILING_OPTIMAL)
3771  continue;
3772 
3773  vk->GetImageSubresourceLayout(hwctx->act_dev, f->img[i], &sub, &layout);
3774  drm_desc->layers[i].planes[0].offset = layout.offset;
3775  drm_desc->layers[i].planes[0].pitch = layout.rowPitch;
3776 
3777  if (hwfctx->flags & AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY)
3778  drm_desc->layers[i].planes[0].offset += f->offset[i];
3779  }
3780 
3781  dst->width = src->width;
3782  dst->height = src->height;
3783  dst->data[0] = (uint8_t *)drm_desc;
3784 
3785  av_log(hwfc, AV_LOG_VERBOSE, "Mapped AVVkFrame to a DRM object!\n");
3786 
3787  return 0;
3788 
3789 end:
3790  av_free(drm_desc);
3791  return err;
3792 }
3793 
3794 #if CONFIG_VAAPI
3795 static int vulkan_map_to_vaapi(AVHWFramesContext *hwfc, AVFrame *dst,
3796  const AVFrame *src, int flags)
3797 {
3798  int err;
3799  AVFrame *tmp = av_frame_alloc();
3800  if (!tmp)
3801  return AVERROR(ENOMEM);
3802 
3803  tmp->format = AV_PIX_FMT_DRM_PRIME;
3804 
3805  err = vulkan_map_to_drm(hwfc, tmp, src, flags);
3806  if (err < 0)
3807  goto fail;
3808 
3809  err = av_hwframe_map(dst, tmp, flags);
3810  if (err < 0)
3811  goto fail;
3812 
3813  err = ff_hwframe_map_replace(dst, src);
3814 
3815 fail:
3816  av_frame_free(&tmp);
3817  return err;
3818 }
3819 #endif
3820 #endif
3821 
3822 static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst,
3823  const AVFrame *src, int flags)
3824 {
3825  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3826 
3827  switch (dst->format) {
3828 #if CONFIG_LIBDRM
3829  case AV_PIX_FMT_DRM_PRIME:
3830  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3831  return vulkan_map_to_drm(hwfc, dst, src, flags);
3832  else
3833  return AVERROR(ENOSYS);
3834 #if CONFIG_VAAPI
3835  case AV_PIX_FMT_VAAPI:
3836  if (p->vkctx.extensions & FF_VK_EXT_DRM_MODIFIER_FLAGS)
3837  return vulkan_map_to_vaapi(hwfc, dst, src, flags);
3838  else
3839  return AVERROR(ENOSYS);
3840 #endif
3841 #endif
3842  default:
3843  break;
3844  }
3845  return AVERROR(ENOSYS);
3846 }
3847 
3848 static int copy_buffer_data(AVHWFramesContext *hwfc, AVBufferRef *buf,
3849  AVFrame *swf, VkBufferImageCopy *region,
3850  int planes, int upload)
3851 {
3852  VkResult ret;
3853  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3854  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3855  AVVulkanDeviceContext *hwctx = &p->p;
3856 
3857  FFVkBuffer *vkbuf = (FFVkBuffer *)buf->data;
3858 
3859  const VkMappedMemoryRange flush_info = {
3860  .sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE,
3861  .memory = vkbuf->mem,
3862  .size = VK_WHOLE_SIZE,
3863  };
3864 
3865  if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && !upload) {
3866  ret = vk->InvalidateMappedMemoryRanges(hwctx->act_dev, 1,
3867  &flush_info);
3868  if (ret != VK_SUCCESS) {
3869  av_log(hwfc, AV_LOG_ERROR, "Failed to invalidate buffer data: %s\n",
3870  ff_vk_ret2str(ret));
3871  return AVERROR_EXTERNAL;
3872  }
3873  }
3874 
3875  for (int i = 0; i < planes; i++)
3876  av_image_copy_plane(vkbuf->mapped_mem + region[i].bufferOffset,
3877  region[i].bufferRowLength,
3878  swf->data[i],
3879  swf->linesize[i],
3880  swf->linesize[i],
3881  region[i].imageExtent.height);
3882 
3883  if (!(vkbuf->flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) && upload) {
3884  ret = vk->FlushMappedMemoryRanges(hwctx->act_dev, 1,
3885  &flush_info);
3886  if (ret != VK_SUCCESS) {
3887  av_log(hwfc, AV_LOG_ERROR, "Failed to flush buffer data: %s\n",
3888  ff_vk_ret2str(ret));
3889  return AVERROR_EXTERNAL;
3890  }
3891  }
3892 
3893  return 0;
3894 }
3895 
3896 static int get_plane_buf(AVHWFramesContext *hwfc, AVBufferRef **dst,
3897  AVFrame *swf, VkBufferImageCopy *region, int upload)
3898 {
3899  int err;
3900  VulkanFramesPriv *fp = hwfc->hwctx;
3901  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3902  const int planes = av_pix_fmt_count_planes(swf->format);
3903 
3904  size_t buf_offset = 0;
3905  for (int i = 0; i < planes; i++) {
3906  size_t size;
3907  ptrdiff_t linesize = swf->linesize[i];
3908 
3909  uint32_t p_w, p_h;
3910  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
3911 
3912  linesize = FFALIGN(linesize,
3913  p->props.properties.limits.optimalBufferCopyRowPitchAlignment);
3914  size = p_h*linesize;
3915 
3916  region[i] = (VkBufferImageCopy) {
3917  .bufferOffset = buf_offset,
3918  .bufferRowLength = linesize,
3919  .bufferImageHeight = p_h,
3920  .imageSubresource.layerCount = 1,
3921  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
3922  /* Rest of the fields adjusted/filled in later */
3923  };
3924 
3925  buf_offset = FFALIGN(buf_offset + size,
3926  p->props.properties.limits.optimalBufferCopyOffsetAlignment);
3927  }
3928 
3929  err = ff_vk_get_pooled_buffer(&p->vkctx, &fp->tmp, dst,
3930  VK_BUFFER_USAGE_TRANSFER_SRC_BIT |
3931  VK_BUFFER_USAGE_TRANSFER_DST_BIT,
3932  NULL, buf_offset,
3933  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT |
3934  VK_MEMORY_PROPERTY_HOST_CACHED_BIT);
3935  if (err < 0)
3936  return err;
3937 
3938  return 0;
3939 }
3940 
3941 static int create_mapped_buffer(AVHWFramesContext *hwfc,
3942  FFVkBuffer *vkb, VkBufferUsageFlags usage,
3943  size_t size,
3944  VkExternalMemoryBufferCreateInfo *create_desc,
3945  VkImportMemoryHostPointerInfoEXT *import_desc,
3946  VkMemoryHostPointerPropertiesEXT props)
3947 {
3948  int err;
3949  VkResult ret;
3950  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
3951  FFVulkanFunctions *vk = &p->vkctx.vkfn;
3952  AVVulkanDeviceContext *hwctx = &p->p;
3953 
3954  VkBufferCreateInfo buf_spawn = {
3955  .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
3956  .pNext = create_desc,
3957  .usage = usage,
3958  .sharingMode = VK_SHARING_MODE_EXCLUSIVE,
3959  .size = size,
3960  };
3961  VkMemoryRequirements req = {
3962  .size = size,
3963  .alignment = p->hprops.minImportedHostPointerAlignment,
3964  .memoryTypeBits = props.memoryTypeBits,
3965  };
3966 
3967  err = ff_vk_alloc_mem(&p->vkctx, &req,
3968  VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT,
3969  import_desc, &vkb->flags, &vkb->mem);
3970  if (err < 0)
3971  return err;
3972 
3973  ret = vk->CreateBuffer(hwctx->act_dev, &buf_spawn, hwctx->alloc, &vkb->buf);
3974  if (ret != VK_SUCCESS) {
3975  vk->FreeMemory(hwctx->act_dev, vkb->mem, hwctx->alloc);
3976  return AVERROR_EXTERNAL;
3977  }
3978 
3979  ret = vk->BindBufferMemory(hwctx->act_dev, vkb->buf, vkb->mem, 0);
3980  if (ret != VK_SUCCESS) {
3981  vk->FreeMemory(hwctx->act_dev, vkb->mem, hwctx->alloc);
3982  vk->DestroyBuffer(hwctx->act_dev, vkb->buf, hwctx->alloc);
3983  return AVERROR_EXTERNAL;
3984  }
3985 
3986  return 0;
3987 }
3988 
3989 static void destroy_avvkbuf(void *opaque, uint8_t *data)
3990 {
3991  FFVulkanContext *s = opaque;
3992  FFVkBuffer *buf = (FFVkBuffer *)data;
3993  ff_vk_free_buf(s, buf);
3994  av_free(buf);
3995 }
3996 
3997 static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs,
3998  AVFrame *swf, VkBufferImageCopy *region, int upload)
3999 {
4000  int err;
4001  VkResult ret;
4002  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4003  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4004  AVVulkanDeviceContext *hwctx = &p->p;
4005 
4006  const int planes = av_pix_fmt_count_planes(swf->format);
4007 
4008  VkExternalMemoryBufferCreateInfo create_desc = {
4009  .sType = VK_STRUCTURE_TYPE_EXTERNAL_MEMORY_BUFFER_CREATE_INFO,
4010  .handleTypes = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
4011  };
4012  VkImportMemoryHostPointerInfoEXT import_desc = {
4013  .sType = VK_STRUCTURE_TYPE_IMPORT_MEMORY_HOST_POINTER_INFO_EXT,
4014  .handleType = VK_EXTERNAL_MEMORY_HANDLE_TYPE_HOST_ALLOCATION_BIT_EXT,
4015  };
4016  VkMemoryHostPointerPropertiesEXT props;
4017 
4018  for (int i = 0; i < planes; i++) {
4019  FFVkBuffer *vkb;
4020  uint32_t p_w, p_h;
4021  size_t offs;
4022  size_t buffer_size;
4023 
4024  /* We can't host map images with negative strides */
4025  if (swf->linesize[i] < 0) {
4026  err = AVERROR(EINVAL);
4027  goto fail;
4028  }
4029 
4030  get_plane_wh(&p_w, &p_h, swf->format, swf->width, swf->height, i);
4031 
4032  /* Get the previous point at which mapping was possible and use it */
4033  offs = (uintptr_t)swf->data[i] % p->hprops.minImportedHostPointerAlignment;
4034  import_desc.pHostPointer = swf->data[i] - offs;
4035 
4036  props = (VkMemoryHostPointerPropertiesEXT) {
4037  VK_STRUCTURE_TYPE_MEMORY_HOST_POINTER_PROPERTIES_EXT,
4038  };
4039  ret = vk->GetMemoryHostPointerPropertiesEXT(hwctx->act_dev,
4040  import_desc.handleType,
4041  import_desc.pHostPointer,
4042  &props);
4043  if (!(ret == VK_SUCCESS && props.memoryTypeBits)) {
4044  err = AVERROR(EINVAL);
4045  goto fail;
4046  }
4047 
4048  /* Buffer region for this plane */
4049  region[i] = (VkBufferImageCopy) {
4050  .bufferOffset = offs,
4051  .bufferRowLength = swf->linesize[i],
4052  .bufferImageHeight = p_h,
4053  .imageSubresource.layerCount = 1,
4054  .imageExtent = (VkExtent3D){ p_w, p_h, 1 },
4055  /* Rest of the fields adjusted/filled in later */
4056  };
4057 
4058  /* Add the offset at the start, which gets ignored */
4059  buffer_size = offs + swf->linesize[i]*p_h;
4060  buffer_size = FFALIGN(buffer_size, p->props.properties.limits.minMemoryMapAlignment);
4061  buffer_size = FFALIGN(buffer_size, p->hprops.minImportedHostPointerAlignment);
4062 
4063  /* Create a buffer */
4064  vkb = av_mallocz(sizeof(*vkb));
4065  if (!vkb) {
4066  err = AVERROR(ENOMEM);
4067  goto fail;
4068  }
4069 
4070  err = create_mapped_buffer(hwfc, vkb,
4071  upload ? VK_BUFFER_USAGE_TRANSFER_SRC_BIT :
4072  VK_BUFFER_USAGE_TRANSFER_DST_BIT,
4073  buffer_size, &create_desc, &import_desc,
4074  props);
4075  if (err < 0) {
4076  av_free(vkb);
4077  goto fail;
4078  }
4079 
4080  /* Create a ref */
4081  dst[*nb_bufs] = av_buffer_create((uint8_t *)vkb, sizeof(*vkb),
4082  destroy_avvkbuf, &p->vkctx, 0);
4083  if (!dst[*nb_bufs]) {
4084  destroy_avvkbuf(&p->vkctx, (uint8_t *)vkb);
4085  err = AVERROR(ENOMEM);
4086  goto fail;
4087  }
4088 
4089  (*nb_bufs)++;
4090  }
4091 
4092  return 0;
4093 
4094 fail:
4095  for (int i = 0; i < (*nb_bufs); i++)
4096  av_buffer_unref(&dst[i]);
4097  return err;
4098 }
4099 
4100 static int vulkan_transfer_frame(AVHWFramesContext *hwfc,
4101  AVFrame *swf, AVFrame *hwf,
4102  int upload)
4103 {
4104  int err;
4105  VulkanFramesPriv *fp = hwfc->hwctx;
4106  VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4107  FFVulkanFunctions *vk = &p->vkctx.vkfn;
4108 
4109  int host_mapped = 0;
4110 
4111  AVVkFrame *hwf_vk = (AVVkFrame *)hwf->data[0];
4112  VkBufferImageCopy region[AV_NUM_DATA_POINTERS]; // always one per plane
4113 
4114  const int planes = av_pix_fmt_count_planes(swf->format);
4115  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(swf->format);
4116  const int nb_images = ff_vk_count_images(hwf_vk);
4117 
4118  VkImageMemoryBarrier2 img_bar[AV_NUM_DATA_POINTERS];
4119  int nb_img_bar = 0;
4120 
4121  AVBufferRef *bufs[AV_NUM_DATA_POINTERS];
4122  int nb_bufs = 0;
4123 
4124  VkCommandBuffer cmd_buf;
4125  FFVkExecContext *exec;
4126 
4127  /* Sanity checking */
4128  if ((swf->format != AV_PIX_FMT_NONE && !av_vkfmt_from_pixfmt(swf->format))) {
4129  av_log(hwfc, AV_LOG_ERROR, "Unsupported software frame pixel format!\n");
4130  return AVERROR(EINVAL);
4131  }
4132 
4133  if (swf->width > hwfc->width || swf->height > hwfc->height)
4134  return AVERROR(EINVAL);
4135 
4136  /* Setup buffers first */
4137  if (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_HOST_MEMORY) {
4138  err = host_map_frame(hwfc, bufs, &nb_bufs, swf, region, upload);
4139  if (err >= 0)
4140  host_mapped = 1;
4141  }
4142 
4143  if (!host_mapped) {
4144  err = get_plane_buf(hwfc, &bufs[0], swf, region, upload);
4145  if (err < 0)
4146  goto end;
4147  nb_bufs = 1;
4148 
4149  if (upload) {
4150  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 1);
4151  if (err < 0)
4152  goto end;
4153  }
4154  }
4155 
4156  exec = ff_vk_exec_get(&p->vkctx, &fp->upload_exec);
4157  cmd_buf = exec->buf;
4158 
4159  ff_vk_exec_start(&p->vkctx, exec);
4160 
4161  /* Prep destination Vulkan frame */
4162  err = ff_vk_exec_add_dep_frame(&p->vkctx, exec, hwf,
4163  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4164  VK_PIPELINE_STAGE_2_TRANSFER_BIT);
4165  if (err < 0)
4166  goto end;
4167 
4168  /* No need to declare buf deps for synchronous transfers (downloads) */
4169  if (upload) {
4170  /* Add the software frame backing the buffers if we're host mapping */
4171  if (host_mapped) {
4172  err = ff_vk_exec_add_dep_sw_frame(&p->vkctx, exec, swf);
4173  if (err < 0) {
4174  ff_vk_exec_discard_deps(&p->vkctx, exec);
4175  goto end;
4176  }
4177  }
4178 
4179  /* Add the buffers as a dependency */
4180  err = ff_vk_exec_add_dep_buf(&p->vkctx, exec, bufs, nb_bufs, 1);
4181  if (err < 0) {
4182  ff_vk_exec_discard_deps(&p->vkctx, exec);
4183  goto end;
4184  }
4185  }
4186 
4187  ff_vk_frame_barrier(&p->vkctx, exec, hwf, img_bar, &nb_img_bar,
4188  VK_PIPELINE_STAGE_2_ALL_COMMANDS_BIT,
4189  VK_PIPELINE_STAGE_2_TRANSFER_BIT_KHR,
4190  upload ? VK_ACCESS_TRANSFER_WRITE_BIT :
4191  VK_ACCESS_TRANSFER_READ_BIT,
4192  upload ? VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL :
4193  VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL,
4194  VK_QUEUE_FAMILY_IGNORED);
4195 
4196  vk->CmdPipelineBarrier2(cmd_buf, &(VkDependencyInfo) {
4197  .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
4198  .pImageMemoryBarriers = img_bar,
4199  .imageMemoryBarrierCount = nb_img_bar,
4200  });
4201 
4202  for (int i = 0; i < planes; i++) {
4203  int buf_idx = FFMIN(i, (nb_bufs - 1));
4204  int img_idx = FFMIN(i, (nb_images - 1));
4205  FFVkBuffer *vkbuf = (FFVkBuffer *)bufs[buf_idx]->data;
4206 
4207  uint32_t orig_stride = region[i].bufferRowLength;
4208  region[i].bufferRowLength /= desc->comp[i].step;
4209  region[i].imageSubresource.aspectMask = ff_vk_aspect_flag(hwf, i);
4210 
4211  if (upload)
4212  vk->CmdCopyBufferToImage(cmd_buf, vkbuf->buf,
4213  hwf_vk->img[img_idx],
4214  img_bar[img_idx].newLayout,
4215  1, &region[i]);
4216  else
4217  vk->CmdCopyImageToBuffer(cmd_buf, hwf_vk->img[img_idx],
4218  img_bar[img_idx].newLayout,
4219  vkbuf->buf,
4220  1, &region[i]);
4221 
4222  region[i].bufferRowLength = orig_stride;
4223  }
4224 
4225  err = ff_vk_exec_submit(&p->vkctx, exec);
4226  if (err < 0) {
4227  ff_vk_exec_discard_deps(&p->vkctx, exec);
4228  } else if (!upload) {
4229  ff_vk_exec_wait(&p->vkctx, exec);
4230  if (!host_mapped)
4231  err = copy_buffer_data(hwfc, bufs[0], swf, region, planes, 0);
4232  }
4233 
4234 end:
4235  for (int i = 0; i < nb_bufs; i++)
4236  av_buffer_unref(&bufs[i]);
4237 
4238  return err;
4239 }
4240 
4241 static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst,
4242  const AVFrame *src)
4243 {
4244  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4245 
4246  switch (src->format) {
4247 #if CONFIG_CUDA
4248  case AV_PIX_FMT_CUDA:
4249 #ifdef _WIN32
4250  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4251  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4252 #else
4253  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4254  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4255 #endif
4256  return vulkan_transfer_data_from_cuda(hwfc, dst, src);
4257 #endif
4258  default:
4259  if (src->hw_frames_ctx)
4260  return AVERROR(ENOSYS);
4261  else
4262  return vulkan_transfer_frame(hwfc, (AVFrame *)src, dst, 1);
4263  }
4264 }
4265 
4266 #if CONFIG_CUDA
4267 static int vulkan_transfer_data_to_cuda(AVHWFramesContext *hwfc, AVFrame *dst,
4268  const AVFrame *src)
4269 {
4270  int err;
4271  CUcontext dummy;
4272  AVVkFrame *dst_f;
4273  AVVkFrameInternal *dst_int;
4274  VulkanFramesPriv *fp = hwfc->hwctx;
4275  const int planes = av_pix_fmt_count_planes(hwfc->sw_format);
4276  const AVPixFmtDescriptor *desc = av_pix_fmt_desc_get(hwfc->sw_format);
4277 
4278  AVHWFramesContext *cuda_fc = (AVHWFramesContext*)dst->hw_frames_ctx->data;
4279  AVHWDeviceContext *cuda_cu = cuda_fc->device_ctx;
4280  AVCUDADeviceContext *cuda_dev = cuda_cu->hwctx;
4281  AVCUDADeviceContextInternal *cu_internal = cuda_dev->internal;
4282  CudaFunctions *cu = cu_internal->cuda_dl;
4283  CUDA_EXTERNAL_SEMAPHORE_WAIT_PARAMS s_w_par[AV_NUM_DATA_POINTERS] = { 0 };
4284  CUDA_EXTERNAL_SEMAPHORE_SIGNAL_PARAMS s_s_par[AV_NUM_DATA_POINTERS] = { 0 };
4285 
4286  dst_f = (AVVkFrame *)src->data[0];
4287 
4288  err = prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_EXPORT);
4289  if (err < 0)
4290  return err;
4291 
4292  err = CHECK_CU(cu->cuCtxPushCurrent(cuda_dev->cuda_ctx));
4293  if (err < 0)
4294  return err;
4295 
4296  err = vulkan_export_to_cuda(hwfc, dst->hw_frames_ctx, src);
4297  if (err < 0) {
4298  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4299  return err;
4300  }
4301 
4302  dst_int = dst_f->internal;
4303 
4304  for (int i = 0; i < planes; i++) {
4305  s_w_par[i].params.fence.value = dst_f->sem_value[i] + 0;
4306  s_s_par[i].params.fence.value = dst_f->sem_value[i] + 1;
4307  }
4308 
4309  err = CHECK_CU(cu->cuWaitExternalSemaphoresAsync(dst_int->cu_sem, s_w_par,
4310  planes, cuda_dev->stream));
4311  if (err < 0)
4312  goto fail;
4313 
4314  for (int i = 0; i < planes; i++) {
4315  CUDA_MEMCPY2D cpy = {
4316  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
4317  .dstDevice = (CUdeviceptr)dst->data[i],
4318  .dstPitch = dst->linesize[i],
4319  .dstY = 0,
4320 
4321  .srcMemoryType = CU_MEMORYTYPE_ARRAY,
4322  .srcArray = dst_int->cu_array[i],
4323  };
4324 
4325  int w, h;
4326  get_plane_wh(&w, &h, hwfc->sw_format, hwfc->width, hwfc->height, i);
4327 
4328  cpy.WidthInBytes = w * desc->comp[i].step;
4329  cpy.Height = h;
4330 
4331  err = CHECK_CU(cu->cuMemcpy2DAsync(&cpy, cuda_dev->stream));
4332  if (err < 0)
4333  goto fail;
4334  }
4335 
4336  err = CHECK_CU(cu->cuSignalExternalSemaphoresAsync(dst_int->cu_sem, s_s_par,
4337  planes, cuda_dev->stream));
4338  if (err < 0)
4339  goto fail;
4340 
4341  for (int i = 0; i < planes; i++)
4342  dst_f->sem_value[i]++;
4343 
4344  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4345 
4346  av_log(hwfc, AV_LOG_VERBOSE, "Transferred Vulkan image to CUDA!\n");
4347 
4348  return prepare_frame(hwfc, &fp->upload_exec, dst_f, PREP_MODE_EXTERNAL_IMPORT);
4349 
4350 fail:
4351  CHECK_CU(cu->cuCtxPopCurrent(&dummy));
4352  vulkan_free_internal(dst_f);
4353  av_buffer_unref(&dst->buf[0]);
4354  return err;
4355 }
4356 #endif
4357 
4358 static int vulkan_transfer_data_from(AVHWFramesContext *hwfc, AVFrame *dst,
4359  const AVFrame *src)
4360 {
4361  av_unused VulkanDevicePriv *p = hwfc->device_ctx->hwctx;
4362 
4363  switch (dst->format) {
4364 #if CONFIG_CUDA
4365  case AV_PIX_FMT_CUDA:
4366 #ifdef _WIN32
4367  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_MEMORY) &&
4368  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_WIN32_SEM))
4369 #else
4370  if ((p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_MEMORY) &&
4371  (p->vkctx.extensions & FF_VK_EXT_EXTERNAL_FD_SEM))
4372 #endif
4373  return vulkan_transfer_data_to_cuda(hwfc, dst, src);
4374 #endif
4375  default:
4376  if (dst->hw_frames_ctx)
4377  return AVERROR(ENOSYS);
4378  else
4379  return vulkan_transfer_frame(hwfc, dst, (AVFrame *)src, 0);
4380  }
4381 }
4382 
4383 static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc,
4384  AVHWFramesContext *src_fc, int flags)
4385 {
4386  return vulkan_frames_init(dst_fc);
4387 }
4388 
4389 AVVkFrame *av_vk_frame_alloc(void)
4390 {
4391  int err;
4392  AVVkFrame *f = av_mallocz(sizeof(AVVkFrame));
4393  if (!f)
4394  return NULL;
4395 
4396  f->internal = av_mallocz(sizeof(*f->internal));
4397  if (!f->internal) {
4398  av_free(f);
4399  return NULL;
4400  }
4401 
4402  err = pthread_mutex_init(&f->internal->update_mutex, NULL);
4403  if (err != 0) {
4404  av_free(f->internal);
4405  av_free(f);
4406  return NULL;
4407  }
4408 
4409  return f;
4410 }
4411 
4412 const HWContextType ff_hwcontext_type_vulkan = {
4413  .type = AV_HWDEVICE_TYPE_VULKAN,
4414  .name = "Vulkan",
4415 
4416  .device_hwctx_size = sizeof(VulkanDevicePriv),
4417  .frames_hwctx_size = sizeof(VulkanFramesPriv),
4418 
4419  .device_init = &vulkan_device_init,
4420  .device_uninit = &vulkan_device_uninit,
4421  .device_create = &vulkan_device_create,
4422  .device_derive = &vulkan_device_derive,
4423 
4424  .frames_get_constraints = &vulkan_frames_get_constraints,
4425  .frames_init = vulkan_frames_init,
4426  .frames_get_buffer = vulkan_get_buffer,
4427  .frames_uninit = vulkan_frames_uninit,
4428 
4429  .transfer_get_formats = vulkan_transfer_get_formats,
4430  .transfer_data_to = vulkan_transfer_data_to,
4431  .transfer_data_from = vulkan_transfer_data_from,
4432 
4433  .map_to = vulkan_map_to,
4434  .map_from = vulkan_map_from,
4435  .frames_derive_to = &vulkan_frames_derive_to,
4436 
4437  .pix_fmts = (const enum AVPixelFormat []) {
4438  AV_PIX_FMT_VULKAN,
4439  AV_PIX_FMT_NONE
4440  },
4441 };
vulkan_loader.h
formats
formats
Definition: signature.h:47
load_libvulkan
static int load_libvulkan(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:536
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:85
FFHWFramesContext::pool_internal
AVBufferPool * pool_internal
Definition: hwcontext_internal.h:101
vulkan_device_init
static int vulkan_device_init(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1696
AV_PIX_FMT_GBRAP16
#define AV_PIX_FMT_GBRAP16
Definition: pixfmt.h:525
FF_ENABLE_DEPRECATION_WARNINGS
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:73
ff_vk_load_props
int ff_vk_load_props(FFVulkanContext *s)
Loads props/mprops/driver_props.
Definition: vulkan.c:106
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:215
AVVulkanDeviceContext::phys_dev
VkPhysicalDevice phys_dev
Physical device.
Definition: hwcontext_vulkan.h:79
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
VulkanDeviceFeatures::vulkan_1_2
VkPhysicalDeviceVulkan12Features vulkan_1_2
Definition: hwcontext_vulkan.c:79
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
VulkanDevicePriv::libvulkan
void * libvulkan
Definition: hwcontext_vulkan.c:104
VulkanOptExtension::name
const char * name
Definition: hwcontext_vulkan.c:571
FFVkFormatEntry::nb_images
int nb_images
Definition: hwcontext_vulkan.c:301
host_map_frame
static int host_map_frame(AVHWFramesContext *hwfc, AVBufferRef **dst, int *nb_bufs, AVFrame *swf, VkBufferImageCopy *region, int upload)
Definition: hwcontext_vulkan.c:3997
AVCUDADeviceContextInternal
Definition: hwcontext_cuda_internal.h:31
AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
@ AV_VK_FRAME_FLAG_DISABLE_MULTIPLANE
Definition: hwcontext_vulkan.h:207
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
hwcontext_cuda_internal.h
FFVulkanExtensions
uint64_t FFVulkanExtensions
Definition: vulkan_functions.h:29
AVBufferPool
The buffer pool.
Definition: buffer_internal.h:88
SET_OLD_QF
#define SET_OLD_QF(field, nb_field, type)
vulkan_transfer_data_to
static int vulkan_transfer_data_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src)
Definition: hwcontext_vulkan.c:4241
FF_VK_EXT_VIDEO_QUEUE
#define FF_VK_EXT_VIDEO_QUEUE
Definition: vulkan_functions.h:53
thread.h
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3170
CHECK_QUEUE
#define CHECK_QUEUE(type, required, fidx, ctx_qf, qc)
AVBufferRef::data
uint8_t * data
The data buffer.
Definition: buffer.h:90
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
FF_VK_EXT_PORTABILITY_SUBSET
#define FF_VK_EXT_PORTABILITY_SUBSET
Definition: vulkan_functions.h:65
ff_vk_qf_init
int ff_vk_qf_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, VkQueueFlagBits dev_family)
Chooses a QF and loads it into a context.
Definition: vulkan.c:232
vulkan_frames_get_constraints
static int vulkan_frames_get_constraints(AVHWDeviceContext *ctx, const void *hwconfig, AVHWFramesConstraints *constraints)
Definition: hwcontext_vulkan.c:2032
AVVkFrameInternal::update_mutex
pthread_mutex_t update_mutex
Definition: hwcontext_vulkan.c:161
FF_VULKAN_DEBUG_PROFILE
@ FF_VULKAN_DEBUG_PROFILE
Definition: hwcontext_vulkan.c:677
av_unused
#define av_unused
Definition: attributes.h:131
vulkan_frames_derive_to
static int vulkan_frames_derive_to(AVHWFramesContext *dst_fc, AVHWFramesContext *src_fc, int flags)
Definition: hwcontext_vulkan.c:4383
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:162
PICK_QF
#define PICK_QF(type, vid_op)
FF_VK_EXT_VIDEO_DECODE_H265
#define FF_VK_EXT_VIDEO_DECODE_H265
Definition: vulkan_functions.h:58
mode
Definition: swscale.c:52
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
pixdesc.h
AVVulkanDeviceContext::get_proc_addr
PFN_vkGetInstanceProcAddr get_proc_addr
Pointer to a vkGetInstanceProcAddr loading function.
Definition: hwcontext_vulkan.h:69
optional_device_exts
static const VulkanOptExtension optional_device_exts[]
Definition: hwcontext_vulkan.c:579
AVFrame::width
int width
Definition: frame.h:461
w
uint8_t w
Definition: llviddspenc.c:38
AV_PIX_FMT_Y216
#define AV_PIX_FMT_Y216
Definition: pixfmt.h:558
create_frame
static int create_frame(AVHWFramesContext *hwfc, AVVkFrame **frame, VkImageTiling tiling, VkImageUsageFlagBits usage, VkImageCreateFlags flags, int nb_layers, void *create_pnext)
Definition: hwcontext_vulkan.c:2397
AVDRMFrameDescriptor::nb_layers
int nb_layers
Number of layers in the frame.
Definition: hwcontext_drm.h:145
AV_PIX_FMT_DRM_PRIME
@ AV_PIX_FMT_DRM_PRIME
DRM-managed buffers exposed through PRIME buffer sharing.
Definition: pixfmt.h:351
AVVulkanFramesContext::create_pnext
void * create_pnext
Extension data for image creation.
Definition: hwcontext_vulkan.h:243
ff_vk_find_struct
static const void * ff_vk_find_struct(const void *chain, VkStructureType stype)
Definition: vulkan.h:326
VulkanDevicePriv::img_qfs
uint32_t img_qfs[5]
Definition: hwcontext_vulkan.c:121
av_hwframe_map
int av_hwframe_map(AVFrame *dst, const AVFrame *src, int flags)
Map a hardware frame.
Definition: hwcontext.c:778
COPY_VAL
#define COPY_VAL(VAL)
nb_vk_formats_list
static const int nb_vk_formats_list
Definition: hwcontext_vulkan.c:391
data
const char data[16]
Definition: mxf.c:149
AVVulkanDeviceContext::queue_family_encode_index
attribute_deprecated int queue_family_encode_index
Queue family index for video encode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:152
AV_PIX_FMT_RGBA128
#define AV_PIX_FMT_RGBA128
Definition: pixfmt.h:580
planes
static const struct @467 planes[]
try_export_flags
static void try_export_flags(AVHWFramesContext *hwfc, VkExternalMemoryHandleTypeFlags *comp_handle_types, VkExternalMemoryHandleTypeFlagBits *iexp, VkExternalMemoryHandleTypeFlagBits exp)
Definition: hwcontext_vulkan.c:2506
AV_PIX_FMT_YUV420P10
#define AV_PIX_FMT_YUV420P10
Definition: pixfmt.h:502
AVVulkanDeviceContext::inst
VkInstance inst
Vulkan instance.
Definition: hwcontext_vulkan.h:74
AV_PIX_FMT_XV30
#define AV_PIX_FMT_XV30
Definition: pixfmt.h:559
AVVulkanFramesContext::lock_frame
void(* lock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Locks a frame, preventing other threads from changing frame properties.
Definition: hwcontext_vulkan.h:288
AV_LOG_VERBOSE
#define AV_LOG_VERBOSE
Detailed information.
Definition: log.h:225
AVVAAPIDeviceContext::display
VADisplay display
The VADisplay handle, to be filled by the user.
Definition: hwcontext_vaapi.h:72
fc
#define fc(width, name, range_min, range_max)
Definition: cbs_av1.c:472
FF_VULKAN_DEBUG_PRACTICES
@ FF_VULKAN_DEBUG_PRACTICES
Definition: hwcontext_vulkan.c:675
vulkan_map_from
static int vulkan_map_from(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:3822
AV_PIX_FMT_BGR24
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition: pixfmt.h:76
AV_PIX_FMT_BGRA
@ AV_PIX_FMT_BGRA
packed BGRA 8:8:8:8, 32bpp, BGRABGRA...
Definition: pixfmt.h:102
FFVkFormatEntry::vkf
VkFormat vkf
Definition: hwcontext_vulkan.c:297
ff_vk_exec_get
FFVkExecContext * ff_vk_exec_get(FFVulkanContext *s, FFVkExecPool *pool)
Retrieve an execution pool.
Definition: vulkan.c:493
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2625
AVDictionary
Definition: dict.c:34
ff_hwframe_map_create
int ff_hwframe_map_create(AVBufferRef *hwframe_ref, AVFrame *dst, const AVFrame *src, void(*unmap)(AVHWFramesContext *ctx, HWMapDescriptor *hwmap), void *priv)
Definition: hwcontext.c:726
av_buffer_ref
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:103
HWMapDescriptor::priv
void * priv
Hardware-specific private data associated with the mapping.
Definition: hwcontext_internal.h:139
FF_VK_EXT_COOP_MATRIX
#define FF_VK_EXT_COOP_MATRIX
Definition: vulkan_functions.h:46
av_popcount
#define av_popcount
Definition: common.h:154
AVDRMFrameDescriptor
DRM frame descriptor.
Definition: hwcontext_drm.h:133
AVHWFramesConstraints::valid_hw_formats
enum AVPixelFormat * valid_hw_formats
A list of possible values for format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:446
AVERROR_UNKNOWN
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:73
AVHWFramesContext::width
int width
The allocated dimensions of the frames in this pool.
Definition: hwcontext.h:217
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
VulkanDeviceSelection::uuid
uint8_t uuid[VK_UUID_SIZE]
Definition: hwcontext_vulkan.c:1135
ff_vk_exec_add_dep_frame
int ff_vk_exec_add_dep_frame(FFVulkanContext *s, FFVkExecContext *e, AVFrame *f, VkPipelineStageFlagBits2 wait_stage, VkPipelineStageFlagBits2 signal_stage)
Definition: vulkan.c:718
FF_VULKAN_DEBUG_PRINTF
@ FF_VULKAN_DEBUG_PRINTF
Definition: hwcontext_vulkan.c:673
AV_PIX_FMT_P212
#define AV_PIX_FMT_P212
Definition: pixfmt.h:568
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:410
FFVkBuffer::buf
VkBuffer buf
Definition: vulkan.h:88
av_image_copy_plane
void av_image_copy_plane(uint8_t *dst, int dst_linesize, const uint8_t *src, int src_linesize, int bytewidth, int height)
Copy image plane from src to dst.
Definition: imgutils.c:374
alloc_mem
static int alloc_mem(AVHWDeviceContext *ctx, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, const void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Definition: hwcontext_vulkan.c:2078
AV_HWDEVICE_TYPE_VULKAN
@ AV_HWDEVICE_TYPE_VULKAN
Definition: hwcontext.h:39
AVHWFramesConstraints
This struct describes the constraints on hardware frames attached to a given device with a hardware-s...
Definition: hwcontext.h:441
AV_HWDEVICE_TYPE_CUDA
@ AV_HWDEVICE_TYPE_CUDA
Definition: hwcontext.h:30
AVDRMDeviceContext::fd
int fd
File descriptor of DRM device.
Definition: hwcontext_drm.h:166
PREP_MODE_DECODING_DPB
@ PREP_MODE_DECODING_DPB
Definition: hwcontext_vulkan.c:2286
FF_VK_EXT_EXTERNAL_FD_SEM
#define FF_VK_EXT_EXTERNAL_FD_SEM
Definition: vulkan_functions.h:34
VulkanDeviceFeatures::device
VkPhysicalDeviceFeatures2 device
Definition: hwcontext_vulkan.c:76
AV_HWFRAME_MAP_WRITE
@ AV_HWFRAME_MAP_WRITE
The mapping must be writeable.
Definition: hwcontext.h:516
VulkanDeviceFeatures::video_maintenance_1
VkPhysicalDeviceVideoMaintenance1FeaturesKHR video_maintenance_1
Definition: hwcontext_vulkan.c:83
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3210
ASPECT_3PLANE
#define ASPECT_3PLANE
Definition: hwcontext_vulkan.c:294
VulkanDevicePriv::hprops
VkPhysicalDeviceExternalMemoryHostPropertiesEXT hprops
Definition: hwcontext_vulkan.c:113
vulkan_device_derive
static int vulkan_device_derive(AVHWDeviceContext *ctx, AVHWDeviceContext *src_ctx, AVDictionary *opts, int flags)
Definition: hwcontext_vulkan.c:1929
VulkanOptExtension::flag
FFVulkanExtensions flag
Definition: hwcontext_vulkan.c:572
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AV_PIX_FMT_GBRP14
#define AV_PIX_FMT_GBRP14
Definition: pixfmt.h:520
AVVkFrame::img
VkImage img[AV_NUM_DATA_POINTERS]
Vulkan images to which the memory is bound to.
Definition: hwcontext_vulkan.h:307
AV_PIX_FMT_GBRAP
@ AV_PIX_FMT_GBRAP
planar GBRA 4:4:4:4 32bpp
Definition: pixfmt.h:212
VulkanDeviceSelection::drm_minor
uint32_t drm_minor
Definition: hwcontext_vulkan.c:1138
lock_frame
static void lock_frame(AVHWFramesContext *fc, AVVkFrame *vkf)
Definition: hwcontext_vulkan.c:2638
fail
#define fail()
Definition: checkasm.h:200
AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY
@ AV_VK_FRAME_FLAG_CONTIGUOUS_MEMORY
Definition: hwcontext_vulkan.h:202
AVDRMLayerDescriptor::nb_planes
int nb_planes
Number of planes in the layer.
Definition: hwcontext_drm.h:106
ff_vk_exec_add_dep_bool_sem
int ff_vk_exec_add_dep_bool_sem(FFVulkanContext *s, FFVkExecContext *e, VkSemaphore *sem, int nb, VkPipelineStageFlagBits2 stage, int wait)
Definition: vulkan.c:647
AV_PIX_FMT_GBRP10
#define AV_PIX_FMT_GBRP10
Definition: pixfmt.h:518
AVVulkanFramesContext::flags
AVVkFrameFlags flags
A combination of AVVkFrameFlags.
Definition: hwcontext_vulkan.h:259
VulkanDevicePriv
Definition: hwcontext_vulkan.c:97
AVVulkanDeviceContext::nb_tx_queues
attribute_deprecated int nb_tx_queues
Definition: hwcontext_vulkan.h:135
AVDRMLayerDescriptor::planes
AVDRMPlaneDescriptor planes[AV_DRM_MAX_PLANES]
Array of planes in this layer.
Definition: hwcontext_drm.h:110
dummy
int dummy
Definition: motion.c:66
AVVkFrame::mem
VkDeviceMemory mem[AV_NUM_DATA_POINTERS]
Memory backing the images.
Definition: hwcontext_vulkan.h:320
device_features_copy_needed
static void device_features_copy_needed(VulkanDeviceFeatures *dst, VulkanDeviceFeatures *src)
Definition: hwcontext_vulkan.c:229
AVVulkanFramesContext
Allocated as AVHWFramesContext.hwctx, used to set pool-specific options.
Definition: hwcontext_vulkan.h:213
av_buffer_pool_init2
AVBufferPool * av_buffer_pool_init2(size_t size, void *opaque, AVBufferRef *(*alloc)(void *opaque, size_t size), void(*pool_free)(void *opaque))
Allocate and initialize a buffer pool with a more complex allocator.
Definition: buffer.c:259
AVHWFramesConstraints::min_width
int min_width
The minimum size of frames in this hw_frames_ctx.
Definition: hwcontext.h:459
lock_queue
static void lock_queue(AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Definition: hwcontext_vulkan.c:1684
AVCUDADeviceContextInternal::cuda_device
CUdevice cuda_device
Definition: hwcontext_cuda_internal.h:34
VulkanDevicePriv::vkctx
FFVulkanContext vkctx
Definition: hwcontext_vulkan.c:106
AV_PIX_FMT_XV48
#define AV_PIX_FMT_XV48
Definition: pixfmt.h:561
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
FF_VK_EXT_VIDEO_ENCODE_H265
#define FF_VK_EXT_VIDEO_ENCODE_H265
Definition: vulkan_functions.h:63
ff_vk_ret2str
const char * ff_vk_ret2str(VkResult res)
Converts Vulkan return values to strings.
Definition: vulkan.c:35
destroy_avvkbuf
static void destroy_avvkbuf(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:3989
AV_PIX_FMT_GRAY16
#define AV_PIX_FMT_GRAY16
Definition: pixfmt.h:486
VulkanDeviceFeatures::vulkan_1_3
VkPhysicalDeviceVulkan13Features vulkan_1_3
Definition: hwcontext_vulkan.c:80
AVVulkanDeviceContext::queue_family_decode_index
attribute_deprecated int queue_family_decode_index
Queue family index for video decode ops, and the amount of queues enabled.
Definition: hwcontext_vulkan.h:162
VulkanDevicePriv::props
VkPhysicalDeviceProperties2 props
Definition: hwcontext_vulkan.c:111
ff_vk_aspect_flag
VkImageAspectFlags ff_vk_aspect_flag(AVFrame *f, int p)
Get the aspect flag for a plane from an image.
Definition: vulkan.c:1306
vk_find_format_entry
static const struct FFVkFormatEntry * vk_find_format_entry(enum AVPixelFormat p)
Definition: hwcontext_vulkan.c:401
AVDRMPlaneDescriptor::offset
ptrdiff_t offset
Offset within that object of this plane.
Definition: hwcontext_drm.h:83
AVHWDeviceContext
This struct aggregates all the (hardware/vendor-specific) "high-level" state, i.e.
Definition: hwcontext.h:60
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:150
AV_PIX_FMT_YUV444P10
#define AV_PIX_FMT_YUV444P10
Definition: pixfmt.h:505
vk_dbg_callback
static VkBool32 VKAPI_CALL vk_dbg_callback(VkDebugUtilsMessageSeverityFlagBitsEXT severity, VkDebugUtilsMessageTypeFlagsEXT messageType, const VkDebugUtilsMessengerCallbackDataEXT *data, void *priv)
Definition: hwcontext_vulkan.c:613
AV_PIX_FMT_Y210
#define AV_PIX_FMT_Y210
Definition: pixfmt.h:556
AVVulkanDeviceQueueFamily::num
int num
Definition: hwcontext_vulkan.h:37
HWContextType::type
enum AVHWDeviceType type
Definition: hwcontext_internal.h:30
ffhwframesctx
static FFHWFramesContext * ffhwframesctx(AVHWFramesContext *ctx)
Definition: hwcontext_internal.h:115
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
check_layers
static int check_layers(AVHWDeviceContext *ctx, AVDictionary *opts, const char *const **dst, uint32_t *num, enum FFVulkanDebugMode *debug_mode)
Definition: hwcontext_vulkan.c:846
AV_PIX_FMT_YUV422P16
#define AV_PIX_FMT_YUV422P16
Definition: pixfmt.h:514
AVVulkanDeviceContext::nb_encode_queues
attribute_deprecated int nb_encode_queues
Definition: hwcontext_vulkan.h:154
AVHWFramesContext::height
int height
Definition: hwcontext.h:217
AVHWFramesConstraints::valid_sw_formats
enum AVPixelFormat * valid_sw_formats
A list of possible values for sw_format in the hw_frames_ctx, terminated by AV_PIX_FMT_NONE.
Definition: hwcontext.h:453
vulkan_map_to
static int vulkan_map_to(AVHWFramesContext *hwfc, AVFrame *dst, const AVFrame *src, int flags)
Definition: hwcontext_vulkan.c:3635
av_dict_get
AVDictionaryEntry * av_dict_get(const AVDictionary *m, const char *key, const AVDictionaryEntry *prev, int flags)
Get a dictionary entry with matching key.
Definition: dict.c:62
av_buffer_pool_get
AVBufferRef * av_buffer_pool_get(AVBufferPool *pool)
Allocate a new AVBuffer, reusing an old buffer from the pool when available.
Definition: buffer.c:390
AV_PIX_FMT_GBRAP10
#define AV_PIX_FMT_GBRAP10
Definition: pixfmt.h:522
AVHWFramesContext::pool
AVBufferPool * pool
A pool from which the frames are allocated by av_hwframe_get_buffer().
Definition: hwcontext.h:178
VulkanDeviceSelection::pci_device
uint32_t pci_device
Definition: hwcontext_vulkan.c:1141
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_PIX_FMT_GBRAP14
#define AV_PIX_FMT_GBRAP14
Definition: pixfmt.h:524
AV_PIX_FMT_GBRAP12
#define AV_PIX_FMT_GBRAP12
Definition: pixfmt.h:523
AV_PIX_FMT_RGB96
#define AV_PIX_FMT_RGB96
Definition: pixfmt.h:579
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:515
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:60
format
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate. The lists are not just lists
ff_vk_load_functions
static int ff_vk_load_functions(AVHWDeviceContext *ctx, FFVulkanFunctions *vk, uint64_t extensions_mask, int has_inst, int has_dev)
Function loader.
Definition: vulkan_loader.h:105
prepare_frame
static int prepare_frame(AVHWFramesContext *hwfc, FFVkExecPool *ectx, AVVkFrame *frame, enum PrepMode pmode)
Definition: hwcontext_vulkan.c:2290
vulkan_transfer_frame
static int vulkan_transfer_frame(AVHWFramesContext *hwfc, AVFrame *swf, AVFrame *hwf, int upload)
Definition: hwcontext_vulkan.c:4100
FF_VK_EXT_DEVICE_DRM
#define FF_VK_EXT_DEVICE_DRM
Definition: vulkan_functions.h:44
ff_vk_exec_wait
void ff_vk_exec_wait(FFVulkanContext *s, FFVkExecContext *e)
Definition: vulkan.c:498
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:178
ASPECT_2PLANE
#define ASPECT_2PLANE
Definition: hwcontext_vulkan.c:293
vulkan_device_uninit
static void vulkan_device_uninit(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1578
AVVulkanFramesContext::unlock_frame
void(* unlock_frame)(struct AVHWFramesContext *fc, AVVkFrame *vkf)
Similar to lock_frame(), unlocks a frame.
Definition: hwcontext_vulkan.h:293
AVVulkanFramesContext::img_flags
VkImageCreateFlags img_flags
Flags to set during image creation.
Definition: hwcontext_vulkan.h:265
VulkanDeviceFeatures::optical_flow
VkPhysicalDeviceOpticalFlowFeaturesNV optical_flow
Definition: hwcontext_vulkan.c:90
vulkan_frames_uninit
static void vulkan_frames_uninit(AVHWFramesContext *hwfc)
Definition: hwcontext_vulkan.c:2648
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:230
AV_PIX_FMT_YUV420P16
#define AV_PIX_FMT_YUV420P16
Definition: pixfmt.h:513
ctx
AVFormatContext * ctx
Definition: movenc.c:49
AVDRMObjectDescriptor::fd
int fd
DRM PRIME fd for the object.
Definition: hwcontext_drm.h:52
AV_PIX_FMT_GRAY14
#define AV_PIX_FMT_GRAY14
Definition: pixfmt.h:485
ff_vk_exec_add_dep_buf
int ff_vk_exec_add_dep_buf(FFVulkanContext *s, FFVkExecContext *e, AVBufferRef **deps, int nb_deps, int ref)
Execution dependency management.
Definition: vulkan.c:573
VulkanDeviceSelection::index
int index
Definition: hwcontext_vulkan.c:1143
AV_PIX_FMT_RGBF32
#define AV_PIX_FMT_RGBF32
Definition: pixfmt.h:576
VulkanFramesPriv::p
AVVulkanFramesContext p
The public AVVulkanFramesContext.
Definition: hwcontext_vulkan.c:144
AV_HWFRAME_MAP_READ
@ AV_HWFRAME_MAP_READ
The mapping must be readable.
Definition: hwcontext.h:512
vulkan_transfer_get_formats
static int vulkan_transfer_get_formats(AVHWFramesContext *hwfc, enum AVHWFrameTransferDirection dir, enum AVPixelFormat **formats)
Definition: hwcontext_vulkan.c:2839
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:242
FFVulkanDebugMode
FFVulkanDebugMode
Definition: hwcontext_vulkan.c:668
AV_PIX_FMT_GRAYF32
#define AV_PIX_FMT_GRAYF32
Definition: pixfmt.h:535
LIBAVUTIL_VERSION_MINOR
#define LIBAVUTIL_VERSION_MINOR
Definition: version.h:82
AV_PIX_FMT_RGBA
@ AV_PIX_FMT_RGBA
packed RGBA 8:8:8:8, 32bpp, RGBARGBA...
Definition: pixfmt.h:100
FFVkFormatEntry::nb_images_fallback
int nb_images_fallback
Definition: hwcontext_vulkan.c:302
vulkan_frame_free_cb
static void vulkan_frame_free_cb(void *opaque, uint8_t *data)
Definition: hwcontext_vulkan.c:2206
AV_PIX_FMT_GRAY10
#define AV_PIX_FMT_GRAY10
Definition: pixfmt.h:483
if
if(ret)
Definition: filter_design.txt:179
av_vkfmt_from_pixfmt
const VkFormat * av_vkfmt_from_pixfmt(enum AVPixelFormat p)
Returns the optimal per-plane Vulkan format for a given sw_format, one for each plane.
Definition: hwcontext_vulkan.c:393
AVVulkanDeviceContext
Main Vulkan context, allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_vulkan.h:59
VulkanDevicePriv::qf_mutex
pthread_mutex_t ** qf_mutex
Definition: hwcontext_vulkan.c:119
AV_PIX_FMT_GBRP16
#define AV_PIX_FMT_GBRP16
Definition: pixfmt.h:521
opts
AVDictionary * opts
Definition: movenc.c:51
PREP_MODE_WRITE
@ PREP_MODE_WRITE
Definition: hwcontext_vulkan.c:2282
AV_PIX_FMT_RGBA64
#define AV_PIX_FMT_RGBA64
Definition: pixfmt.h:492
ff_vk_alloc_mem
int ff_vk_alloc_mem(FFVulkanContext *s, VkMemoryRequirements *req, VkMemoryPropertyFlagBits req_flags, void *alloc_extension, VkMemoryPropertyFlagBits *mem_flags, VkDeviceMemory *mem)
Memory/buffer/image allocation helpers.
Definition: vulkan.c:899
NULL
#define NULL
Definition: coverity.c:32
pick_video_queue_family
static int pick_video_queue_family(VkQueueFamilyProperties2 *qf, VkQueueFamilyVideoPropertiesKHR *qf_vid, uint32_t num_qf, VkVideoCodecOperationFlagBitsKHR flags)
Definition: hwcontext_vulkan.c:1344
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:210
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
FF_VK_EXT_DRM_MODIFIER_FLAGS
#define FF_VK_EXT_DRM_MODIFIER_FLAGS
Definition: vulkan_functions.h:32
av_buffer_unref
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition: buffer.c:139
AVVulkanDeviceContext::nb_enabled_dev_extensions
int nb_enabled_dev_extensions
Definition: hwcontext_vulkan.h:113
FFVkFormatEntry
Definition: hwcontext_vulkan.c:296
FF_VK_EXT_SHADER_OBJECT
#define FF_VK_EXT_SHADER_OBJECT
Definition: vulkan_functions.h:48
AV_PIX_FMT_YUYV422
@ AV_PIX_FMT_YUYV422
packed YUV 4:2:2, 16bpp, Y0 Cb Y1 Cr
Definition: pixfmt.h:74
AVVkFrameInternal
Definition: hwcontext_vulkan.c:160
FF_VK_EXT_VIDEO_ENCODE_QUEUE
#define FF_VK_EXT_VIDEO_ENCODE_QUEUE
Definition: vulkan_functions.h:61
VulkanDevicePriv::debug_ctx
VkDebugUtilsMessengerEXT debug_ctx
Definition: hwcontext_vulkan.c:125
FF_VULKAN_DEBUG_NONE
@ FF_VULKAN_DEBUG_NONE
Definition: hwcontext_vulkan.c:669
AVVulkanFramesContext::alloc_pnext
void * alloc_pnext[AV_NUM_DATA_POINTERS]
Extension data for memory allocation.
Definition: hwcontext_vulkan.h:252
setup_queue_families
static int setup_queue_families(AVHWDeviceContext *ctx, VkDeviceCreateInfo *cd)
Definition: hwcontext_vulkan.c:1373
AVVulkanDeviceContext::unlock_queue
void(* unlock_queue)(struct AVHWDeviceContext *ctx, uint32_t queue_family, uint32_t index)
Similar to lock_queue(), unlocks a queue.
Definition: hwcontext_vulkan.h:178
LIBAVUTIL_VERSION_MAJOR
#define LIBAVUTIL_VERSION_MAJOR
Definition: version.h:81
AVVulkanDeviceContext::nb_decode_queues
attribute_deprecated int nb_decode_queues
Definition: hwcontext_vulkan.h:164
AV_PIX_FMT_P410
#define AV_PIX_FMT_P410
Definition: pixfmt.h:567
av_buffer_pool_uninit
void av_buffer_pool_uninit(AVBufferPool **ppool)
Mark the pool as being available for freeing.
Definition: buffer.c:328
AVVulkanDeviceContext::nb_qf
int nb_qf
Definition: hwcontext_vulkan.h:189
ff_hwcontext_type_vulkan
const HWContextType ff_hwcontext_type_vulkan
Definition: hwcontext_vulkan.c:4412
hwcontext_vulkan.h
AVVulkanDeviceContext::queue_family_tx_index
attribute_deprecated int queue_family_tx_index
Queue family index for transfer operations and the number of queues enabled.
Definition: hwcontext_vulkan.h:133
AVVulkanDeviceContext::enabled_inst_extensions
const char *const * enabled_inst_extensions
Enabled instance extensions.
Definition: hwcontext_vulkan.h:101
vk_formats_list
static const struct FFVkFormatEntry vk_formats_list[]
AVVulkanFramesContext::format
VkFormat format[AV_NUM_DATA_POINTERS]
Vulkan format for each image.
Definition: hwcontext_vulkan.h:273
AVVulkanFramesContext::usage
VkImageUsageFlagBits usage
Defines extra usage of output frames.
Definition: hwcontext_vulkan.h:232
AV_PIX_FMT_BGR0
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition: pixfmt.h:265
AV_PIX_FMT_YUV422P10
#define AV_PIX_FMT_YUV422P10
Definition: pixfmt.h:503
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:306
alloc_bind_mem
static int alloc_bind_mem(AVHWFramesContext *hwfc, AVVkFrame *f, void *alloc_pnext, size_t alloc_pnext_stride)
Definition: hwcontext_vulkan.c:2211
AVVulkanDeviceContext::qf
AVVulkanDeviceQueueFamily qf[64]
Queue families used.
Definition: hwcontext_vulkan.h:188
AV_PIX_FMT_GRAY8
@ AV_PIX_FMT_GRAY8
Y , 8bpp.
Definition: pixfmt.h:81
FFVulkanContext
Definition: vulkan.h:271
exp
int8_t exp
Definition: eval.c:73
VulkanFramesPriv
Definition: hwcontext_vulkan.c:140
vulkan_frame_free
static void vulkan_frame_free(AVHWFramesContext *hwfc, AVVkFrame *f)
Definition: hwcontext_vulkan.c:2172
index
int index
Definition: gxfenc.c:90
pthread_mutex_unlock
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:82
av_buffer_create
AVBufferRef * av_buffer_create(uint8_t *data, size_t size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:55
vkfmt_from_pixfmt2
static int vkfmt_from_pixfmt2(AVHWDeviceContext *dev_ctx, enum AVPixelFormat p, VkImageTiling tiling, VkFormat fmts[AV_NUM_DATA_POINTERS], int *nb_images, VkImageAspectFlags *aspect, VkImageUsageFlags *supported_usage, int disable_multiplane, int need_storage)
Definition: hwcontext_vulkan.c:443
VulkanDeviceSelection
Definition: hwcontext_vulkan.c:1134
source
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a source
Definition: filter_design.txt:255
VulkanDevicePriv::nb_tot_qfs
uint32_t nb_tot_qfs
Definition: hwcontext_vulkan.c:120
FFVulkanContext::device
AVHWDeviceContext * device
Definition: vulkan.h:299
VulkanDeviceFeatures
Definition: hwcontext_vulkan.c:75
AVDRMFrameDescriptor::layers
AVDRMLayerDescriptor layers[AV_DRM_MAX_PLANES]
Array of layers in the frame.
Definition: hwcontext_drm.h:149
usage
const char * usage
Definition: floatimg_cmp.c:60
PREP_MODE_DECODING_DST
@ PREP_MODE_DECODING_DST
Definition: hwcontext_vulkan.c:2285
AVVkFrame::size
size_t size[AV_NUM_DATA_POINTERS]
Definition: hwcontext_vulkan.h:321
vulkan_pool_alloc
static AVBufferRef * vulkan_pool_alloc(void *opaque, size_t size)
Definition: hwcontext_vulkan.c:2568
AV_PIX_FMT_X2BGR10
#define AV_PIX_FMT_X2BGR10
Definition: pixfmt.h:564
PrepMode
PrepMode
Definition: hwcontext_vulkan.c:2280
FF_VK_EXT_VIDEO_MAINTENANCE_1
#define FF_VK_EXT_VIDEO_MAINTENANCE_1
Definition: vulkan_functions.h:54
VulkanDeviceSelection::has_drm
uint32_t has_drm
Definition: hwcontext_vulkan.c:1139
f
f
Definition: af_crystalizer.c:122
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
AV_PIX_FMT_RGB24
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition: pixfmt.h:75
VulkanDeviceFeatures::vulkan_1_1
VkPhysicalDeviceVulkan11Features vulkan_1_1
Definition: hwcontext_vulkan.c:78
sem_wait
#define sem_wait(psem)
Definition: semaphore.h:27
AV_PIX_FMT_P012
#define AV_PIX_FMT_P012
Definition: pixfmt.h:553
AV_PIX_FMT_FLAG_RGB
#define AV_PIX_FMT_FLAG_RGB
The pixel format contains RGB-like data (as opposed to YUV/grayscale).
Definition: pixdesc.h:136
dst
uint8_t ptrdiff_t const uint8_t ptrdiff_t int intptr_t intptr_t int int16_t * dst
Definition: dsp.h:83
AVVkFrame
Definition: hwcontext_vulkan.h:302
av_vk_frame_alloc
AVVkFrame * av_vk_frame_alloc(void)
Allocates a single AVVkFrame and initializes everything as 0.
Definition: hwcontext_vulkan.c:4389
FF_VULKAN_DEBUG_VALIDATE
@ FF_VULKAN_DEBUG_VALIDATE
Definition: hwcontext_vulkan.c:671
vulkan.h
av_err2str
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:122
FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
#define FF_VK_EXT_EXTERNAL_DMABUF_MEMORY
Definition: vulkan_functions.h:31
vulkan_device_free
static void vulkan_device_free(AVHWDeviceContext *ctx)
Definition: hwcontext_vulkan.c:1555
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425