FFmpeg
vf_flip_vulkan.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2021 Wu Jianhua <jianhua.wu@intel.com>
3  * Copyright (c) Lynne
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/random_seed.h"
23 #include "libavutil/opt.h"
24 #include "vulkan_filter.h"
25 #include "vulkan_spirv.h"
26 #include "internal.h"
27 
28 enum FlipType {
32 };
33 
34 typedef struct FlipVulkanContext {
36 
42  VkSampler sampler;
44 
46 {
47  int err = 0;
48  uint8_t *spv_data;
49  size_t spv_len;
50  void *spv_opaque = NULL;
51  FlipVulkanContext *s = ctx->priv;
52  FFVulkanContext *vkctx = &s->vkctx;
53  const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
54  FFVkSPIRVShader *shd = &s->shd;
55  FFVkSPIRVCompiler *spv;
57 
58  spv = ff_vk_spirv_init();
59  if (!spv) {
60  av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
61  return AVERROR_EXTERNAL;
62  }
63 
64  ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
65  RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0, 0, 0, NULL));
66  RET(ff_vk_init_sampler(vkctx, &s->sampler, 1, VK_FILTER_LINEAR));
67  RET(ff_vk_shader_init(&s->pl, &s->shd, "flip_compute",
68  VK_SHADER_STAGE_COMPUTE_BIT, 0));
69 
70  ff_vk_shader_set_compute_sizes(&s->shd, 32, 32, 1);
71 
73  {
74  .name = "input_image",
75  .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
76  .dimensions = 2,
77  .elems = planes,
78  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
79  .samplers = DUP_SAMPLER(s->sampler),
80  },
81  {
82  .name = "output_image",
83  .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
84  .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format),
85  .mem_quali = "writeonly",
86  .dimensions = 2,
87  .elems = planes,
88  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
89  },
90  };
91 
92  RET(ff_vk_pipeline_descriptor_set_add(vkctx, &s->pl, shd, desc, 2, 0, 0));
93 
94  GLSLC(0, void main() );
95  GLSLC(0, { );
96  GLSLC(1, ivec2 size; );
97  GLSLC(1, const ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
98  for (int i = 0; i < planes; i++) {
99  GLSLC(0, );
100  GLSLF(1, size = imageSize(output_image[%i]); ,i);
101  GLSLC(1, if (IS_WITHIN(pos, size)) { );
102  switch (type)
103  {
104  case FLIP_HORIZONTAL:
105  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(size.x - pos.x, pos.y)); ,i);
106  break;
107  case FLIP_VERTICAL:
108  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(pos.x, size.y - pos.y)); ,i);
109  break;
110  case FLIP_BOTH:
111  GLSLF(2, vec4 res = texture(input_image[%i], ivec2(size.xy - pos.xy));, i);
112  break;
113  default:
114  GLSLF(2, vec4 res = texture(input_image[%i], pos); ,i);
115  break;
116  }
117  GLSLF(2, imageStore(output_image[%i], pos, res); ,i);
118  GLSLC(1, } );
119  }
120  GLSLC(0, } );
121 
122  RET(spv->compile_shader(spv, ctx, shd, &spv_data, &spv_len, "main",
123  &spv_opaque));
124  RET(ff_vk_shader_create(vkctx, shd, spv_data, spv_len, "main"));
125 
126  RET(ff_vk_init_compute_pipeline(vkctx, &s->pl, shd));
127  RET(ff_vk_exec_pipeline_register(vkctx, &s->e, &s->pl));
128 
129  s->initialized = 1;
130 
131 fail:
132  if (spv_opaque)
133  spv->free_shader(spv, &spv_opaque);
134  if (spv)
135  spv->uninit(&spv);
136 
137  return err;
138 }
139 
141 {
142  FlipVulkanContext *s = avctx->priv;
143 
144  FFVulkanContext *vkctx = &s->vkctx;
145  FFVulkanFunctions *vk = &vkctx->vkfn;
146 
147  ff_vk_exec_pool_free(vkctx, &s->e);
148  ff_vk_pipeline_free(vkctx, &s->pl);
149  ff_vk_shader_free(vkctx, &s->shd);
150 
151  if (s->sampler)
152  vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
153  vkctx->hwctx->alloc);
154 
155  ff_vk_uninit(&s->vkctx);
156 
157  s->initialized = 0;
158 }
159 
161 {
162  int err;
163  AVFrame *out = NULL;
164  AVFilterContext *ctx = link->dst;
165  FlipVulkanContext *s = ctx->priv;
166  AVFilterLink *outlink = ctx->outputs[0];
167 
168  out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
169  if (!out) {
170  err = AVERROR(ENOMEM);
171  goto fail;
172  }
173 
174  if (!s->initialized)
175  RET(init_filter(ctx, in, type));
176 
177  RET(ff_vk_filter_process_simple(&s->vkctx, &s->e, &s->pl, out, in,
178  s->sampler, NULL, 0));
179 
181 
182  av_frame_free(&in);
183 
184  return ff_filter_frame(outlink, out);
185 
186 fail:
187  av_frame_free(&in);
188  av_frame_free(&out);
189  return err;
190 }
191 
193 {
194  return filter_frame(link, in, FLIP_HORIZONTAL);
195 }
196 
198 {
199  return filter_frame(link, in, FLIP_VERTICAL);
200 }
201 
203 {
204  return filter_frame(link, in, FLIP_BOTH);
205 }
206 
208  {
209  .name = "default",
210  .type = AVMEDIA_TYPE_VIDEO,
211  .config_props = &ff_vk_filter_config_output,
212  }
213 };
214 
215 static const AVOption hflip_vulkan_options[] = {
216  { NULL },
217 };
218 
219 AVFILTER_DEFINE_CLASS(hflip_vulkan);
220 
222  {
223  .name = "default",
224  .type = AVMEDIA_TYPE_VIDEO,
225  .filter_frame = &hflip_vulkan_filter_frame,
226  .config_props = &ff_vk_filter_config_input,
227  }
228 };
229 
231  .name = "hflip_vulkan",
232  .description = NULL_IF_CONFIG_SMALL("Horizontally flip the input video in Vulkan"),
233  .priv_size = sizeof(FlipVulkanContext),
239  .priv_class = &hflip_vulkan_class,
240  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
241 };
242 
243 static const AVOption vflip_vulkan_options[] = {
244  { NULL },
245 };
246 
247 AVFILTER_DEFINE_CLASS(vflip_vulkan);
248 
250  {
251  .name = "default",
252  .type = AVMEDIA_TYPE_VIDEO,
253  .filter_frame = &vflip_vulkan_filter_frame,
254  .config_props = &ff_vk_filter_config_input,
255  }
256 };
257 
259  .name = "vflip_vulkan",
260  .description = NULL_IF_CONFIG_SMALL("Vertically flip the input video in Vulkan"),
261  .priv_size = sizeof(FlipVulkanContext),
267  .priv_class = &vflip_vulkan_class,
268  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
269 };
270 
271 static const AVOption flip_vulkan_options[] = {
272  { NULL },
273 };
274 
275 AVFILTER_DEFINE_CLASS(flip_vulkan);
276 
277 static const AVFilterPad flip_vulkan_inputs[] = {
278  {
279  .name = "default",
280  .type = AVMEDIA_TYPE_VIDEO,
281  .filter_frame = &flip_vulkan_filter_frame,
282  .config_props = &ff_vk_filter_config_input,
283  }
284 };
285 
287  .name = "flip_vulkan",
288  .description = NULL_IF_CONFIG_SMALL("Flip both horizontally and vertically"),
289  .priv_size = sizeof(FlipVulkanContext),
295  .priv_class = &flip_vulkan_class,
296  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
297  .flags = AVFILTER_FLAG_HWDEVICE,
298 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:101
ff_vk_pipeline_free
void ff_vk_pipeline_free(FFVulkanContext *s, FFVulkanPipeline *pl)
Definition: vulkan.c:1842
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:54
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: internal.h:374
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:971
ff_vk_qf_init
int ff_vk_qf_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, VkQueueFlagBits dev_family)
Chooses a QF and loads it into a context.
Definition: vulkan.c:202
flip_vulkan_filter_frame
static int flip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:202
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:100
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:330
ff_vk_filter_init
int ff_vk_filter_init(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan_filter.c:214
ff_vk_shader_create
int ff_vk_shader_create(FFVulkanContext *s, FFVkSPIRVShader *shd, uint8_t *spirv, size_t spirv_size, const char *entrypoint)
Definition: vulkan.c:1387
AVOption
AVOption.
Definition: opt.h:251
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:1870
FFVkSPIRVCompiler::uninit
void(* uninit)(struct FFVkSPIRVCompiler **ctx)
Definition: vulkan_spirv.h:33
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
ff_vk_pipeline_descriptor_set_add
int ff_vk_pipeline_descriptor_set_add(FFVulkanContext *s, FFVulkanPipeline *pl, FFVkSPIRVShader *shd, FFVulkanDescriptorSetBinding *desc, int nb, int read_only, int print_to_shader_only)
Add descriptor to a pipeline.
Definition: vulkan.c:1437
ff_vk_shader_set_compute_sizes
void ff_vk_shader_set_compute_sizes(FFVkSPIRVShader *shd, int x, int y, int z)
Definition: vulkan.c:1345
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:376
ff_vf_vflip_vulkan
const AVFilter ff_vf_vflip_vulkan
Definition: vf_flip_vulkan.c:258
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2976
FlipType
FlipType
Definition: vf_flip_vulkan.c:28
vflip_vulkan_options
static const AVOption vflip_vulkan_options[]
Definition: vf_flip_vulkan.c:243
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:48
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:412
fail
#define fail()
Definition: checkasm.h:137
vulkan_filter.h
type
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf type
Definition: writing_filters.txt:86
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:49
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
ff_vf_hflip_vulkan
const AVFilter ff_vf_hflip_vulkan
Definition: vf_flip_vulkan.c:230
s
#define s(width, name)
Definition: cbs_vp9.c:256
FlipVulkanContext::initialized
int initialized
Definition: vf_flip_vulkan.c:37
vflip_vulkan_inputs
static const AVFilterPad vflip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:249
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts_bsf.c:365
ctx
AVFormatContext * ctx
Definition: movenc.c:48
FFVkSPIRVCompiler::compile_shader
int(* compile_shader)(struct FFVkSPIRVCompiler *ctx, void *avctx, struct FFVkSPIRVShader *shd, uint8_t **data, size_t *size, const char *entrypoint, void **opaque)
Definition: vulkan_spirv.h:29
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:233
hflip_vulkan_inputs
static const AVFilterPad hflip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:221
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:194
FlipVulkanContext::sampler
VkSampler sampler
Definition: vf_flip_vulkan.c:42
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:736
init_filter
static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in, enum FlipType type)
Definition: vf_flip_vulkan.c:45
FLIP_VERTICAL
@ FLIP_VERTICAL
Definition: vf_flip_vulkan.c:29
ff_vk_filter_config_output
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan_filter.c:191
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *in, enum FlipType type)
Definition: vf_flip_vulkan.c:160
ff_vk_init_compute_pipeline
int ff_vk_init_compute_pipeline(FFVulkanContext *s, FFVulkanPipeline *pl, FFVkSPIRVShader *shd)
Definition: vulkan.c:1783
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:273
FFVulkanContext
Definition: vulkan.h:229
FFVulkanPipeline
Definition: vulkan.h:132
FlipVulkanContext::qf
FFVkQueueFamilyCtx qf
Definition: vf_flip_vulkan.c:40
FlipVulkanContext::e
FFVkExecPool e
Definition: vf_flip_vulkan.c:39
ff_vk_shader_init
int ff_vk_shader_init(FFVulkanPipeline *pl, FFVkSPIRVShader *shd, const char *name, VkShaderStageFlags stage, uint32_t required_subgroup_size)
Shader management.
Definition: vulkan.c:1319
main
int main(int argc, char **argv)
Definition: avio_http_serve_files.c:99
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(hflip_vulkan)
FFVulkanDescriptorSetBinding
Definition: vulkan.h:84
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:114
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:138
FLIP_BOTH
@ FLIP_BOTH
Definition: vf_flip_vulkan.c:31
flip_vulkan_inputs
static const AVFilterPad flip_vulkan_inputs[]
Definition: vf_flip_vulkan.c:277
size
int size
Definition: twinvq_data.h:10344
FFVkQueueFamilyCtx
Definition: vulkan.h:111
FLIP_HORIZONTAL
@ FLIP_HORIZONTAL
Definition: vf_flip_vulkan.c:30
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
hflip_vulkan_filter_frame
static int hflip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:192
internal.h
ff_vk_filter_process_simple
int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e, FFVulkanPipeline *pl, AVFrame *out_f, AVFrame *in_f, VkSampler sampler, void *push_src, size_t push_size)
Submit a compute shader with a zero/one input and single out for execution.
Definition: vulkan_filter.c:223
FFVkSPIRVCompiler
Definition: vulkan_spirv.h:27
planes
static const struct @352 planes[]
flip_vulkan_uninit
static av_cold void flip_vulkan_uninit(AVFilterContext *avctx)
Definition: vf_flip_vulkan.c:140
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:184
vflip_vulkan_filter_frame
static int vflip_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_flip_vulkan.c:197
FlipVulkanContext::shd
FFVkSPIRVShader shd
Definition: vf_flip_vulkan.c:41
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
FlipVulkanContext::vkctx
FFVulkanContext vkctx
Definition: vf_flip_vulkan.c:35
hflip_vulkan_options
static const AVOption hflip_vulkan_options[]
Definition: vf_flip_vulkan.c:215
DUP_SAMPLER
#define DUP_SAMPLER(x)
Definition: vulkan.h:74
FlipVulkanContext::pl
FFVulkanPipeline pl
Definition: vf_flip_vulkan.c:38
ff_vk_shader_rep_fmt
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pixfmt)
Returns the format to use for images in shaders.
Definition: vulkan.c:1181
ff_vf_flip_vulkan
const AVFilter ff_vf_flip_vulkan
Definition: vf_flip_vulkan.c:286
vulkan_spirv.h
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:55
GLSLF
#define GLSLF(N, S,...)
Definition: vulkan.h:55
flip_vulkan_outputs
static const AVFilterPad flip_vulkan_outputs[]
Definition: vf_flip_vulkan.c:207
FFVkSPIRVCompiler::free_shader
void(* free_shader)(struct FFVkSPIRVCompiler *ctx, void **opaque)
Definition: vulkan_spirv.h:32
AVFilter
Filter definition.
Definition: avfilter.h:166
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:232
FFVkExecPool
Definition: vulkan.h:210
pos
unsigned int pos
Definition: spdifenc.c:413
random_seed.h
FFVkSPIRVShader
Definition: vulkan.h:76
AVFilterContext
An instance of a filter.
Definition: avfilter.h:397
desc
const char * desc
Definition: libsvtav1.c:83
GLSLC
#define GLSLC(N, S)
Definition: vulkan.h:45
ff_vk_filter_config_input
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan_filter.c:160
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanContext::hwctx
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:250
FlipVulkanContext
Definition: vf_flip_vulkan.c:34
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:70
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:195
ff_vk_init_sampler
int ff_vk_init_sampler(FFVulkanContext *s, VkSampler *sampler, int unnorm_coords, VkFilter filt)
Create a sampler.
Definition: vulkan.c:1137
ff_vk_exec_pipeline_register
int ff_vk_exec_pipeline_register(FFVulkanContext *s, FFVkExecPool *pool, FFVulkanPipeline *pl)
Register a pipeline with an exec pool.
Definition: vulkan.c:1551
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
uninit
static av_cold int uninit(AVCodecContext *avctx)
Definition: crystalhd.c:285
ff_vk_shader_free
void ff_vk_shader_free(FFVulkanContext *s, FFVkSPIRVShader *shd)
Definition: vulkan.c:1378
RET
#define RET(x)
Definition: vulkan.h:68
FFVulkanFunctions
Definition: vulkan_functions.h:224
flip_vulkan_options
static const AVOption flip_vulkan_options[]
Definition: vf_flip_vulkan.c:271