FFmpeg
vf_chromaber_vulkan.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) Lynne
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include "libavutil/random_seed.h"
22 #include "libavutil/opt.h"
23 #include "libavutil/vulkan_spirv.h"
24 #include "vulkan_filter.h"
25 
26 #include "filters.h"
27 #include "video.h"
28 
31 
36  VkSampler sampler;
37 
38  /* Push constants / options */
39  struct {
40  float dist[2];
41  } opts;
43 
44 static const char distort_chroma_kernel[] = {
45  C(0, void distort_rgb(ivec2 size, ivec2 pos) )
46  C(0, { )
47  C(1, const vec2 p = ((vec2(pos)/vec2(size)) - 0.5f)*2.0f; )
48  C(1, const vec2 o = p * (dist - 1.0f); )
49  C(0, )
50  C(1, vec4 res; )
51  C(1, res.r = texture(input_img[0], ((p - o)/2.0f) + 0.5f).r; )
52  C(1, res.g = texture(input_img[0], ((p )/2.0f) + 0.5f).g; )
53  C(1, res.b = texture(input_img[0], ((p + o)/2.0f) + 0.5f).b; )
54  C(1, res.a = texture(input_img[0], ((p )/2.0f) + 0.5f).a; )
55  C(1, imageStore(output_img[0], pos, res); )
56  C(0, } )
57  C(0, )
58  C(0, void distort_chroma(int idx, ivec2 size, ivec2 pos) )
59  C(0, { )
60  C(1, vec2 p = ((vec2(pos)/vec2(size)) - 0.5f)*2.0f; )
61  C(1, float d = sqrt(p.x*p.x + p.y*p.y); )
62  C(1, p *= d / (d*dist); )
63  C(1, vec4 res = texture(input_img[idx], (p/2.0f) + 0.5f); )
64  C(1, imageStore(output_img[idx], pos, res); )
65  C(0, } )
66 };
67 
69 {
70  int err;
71  uint8_t *spv_data;
72  size_t spv_len;
73  void *spv_opaque = NULL;
75  FFVulkanContext *vkctx = &s->vkctx;
76  const int planes = av_pix_fmt_count_planes(s->vkctx.output_format);
77  FFVulkanShader *shd = &s->shd;
78  FFVkSPIRVCompiler *spv;
80 
81  /* Normalize options */
82  s->opts.dist[0] = (s->opts.dist[0] / 100.0f) + 1.0f;
83  s->opts.dist[1] = (s->opts.dist[1] / 100.0f) + 1.0f;
84 
85  spv = ff_vk_spirv_init();
86  if (!spv) {
87  av_log(ctx, AV_LOG_ERROR, "Unable to initialize SPIR-V compiler!\n");
88  return AVERROR_EXTERNAL;
89  }
90 
91  ff_vk_qf_init(vkctx, &s->qf, VK_QUEUE_COMPUTE_BIT);
92  RET(ff_vk_exec_pool_init(vkctx, &s->qf, &s->e, s->qf.nb_queues*4, 0, 0, 0, NULL));
93  RET(ff_vk_init_sampler(vkctx, &s->sampler, 0, VK_FILTER_LINEAR));
94  RET(ff_vk_shader_init(vkctx, &s->shd, "chromatic_abberation",
95  VK_SHADER_STAGE_COMPUTE_BIT,
96  NULL, 0,
97  32, 32, 1,
98  0));
99 
100  GLSLC(0, layout(push_constant, std430) uniform pushConstants { );
101  GLSLC(1, vec2 dist; );
102  GLSLC(0, }; );
103  GLSLC(0, );
104 
105  ff_vk_shader_add_push_const(&s->shd, 0, sizeof(s->opts),
106  VK_SHADER_STAGE_COMPUTE_BIT);
107 
109  {
110  .name = "input_img",
111  .type = VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER,
112  .dimensions = 2,
113  .elems = planes,
114  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
115  .samplers = DUP_SAMPLER(s->sampler),
116  },
117  {
118  .name = "output_img",
119  .type = VK_DESCRIPTOR_TYPE_STORAGE_IMAGE,
120  .mem_layout = ff_vk_shader_rep_fmt(s->vkctx.output_format, FF_VK_REP_FLOAT),
121  .mem_quali = "writeonly",
122  .dimensions = 2,
123  .elems = planes,
124  .stages = VK_SHADER_STAGE_COMPUTE_BIT,
125  },
126  };
127 
128  RET(ff_vk_shader_add_descriptor_set(vkctx, &s->shd, desc, 2, 0, 0));
129 
131  GLSLC(0, void main() );
132  GLSLC(0, { );
133  GLSLC(1, ivec2 pos = ivec2(gl_GlobalInvocationID.xy); );
134  if (planes == 1) {
135  GLSLC(1, distort_rgb(imageSize(output_img[0]), pos); );
136  } else {
137  GLSLC(1, ivec2 size = imageSize(output_img[0]); );
138  GLSLC(1, vec2 npos = vec2(pos)/vec2(size); );
139  GLSLC(1, vec4 res = texture(input_img[0], npos); );
140  GLSLC(1, imageStore(output_img[0], pos, res); );
141  for (int i = 1; i < planes; i++) {
142  GLSLC(0, );
143  GLSLF(1, size = imageSize(output_img[%i]); ,i);
144  GLSLC(1, if (!IS_WITHIN(pos, size)) );
145  GLSLC(2, return; );
146  GLSLF(1, distort_chroma(%i, size, pos); ,i);
147  }
148  }
149  GLSLC(0, } );
150 
151  RET(spv->compile_shader(vkctx, spv, shd, &spv_data, &spv_len, "main",
152  &spv_opaque));
153  RET(ff_vk_shader_link(vkctx, shd, spv_data, spv_len, "main"));
154 
155  RET(ff_vk_shader_register_exec(vkctx, &s->e, &s->shd));
156 
157  s->initialized = 1;
158 
159 fail:
160  if (spv_opaque)
161  spv->free_shader(spv, &spv_opaque);
162  if (spv)
163  spv->uninit(&spv);
164 
165  return err;
166 }
167 
169 {
170  int err;
171  AVFilterContext *ctx = link->dst;
173  AVFilterLink *outlink = ctx->outputs[0];
174 
175  AVFrame *out = ff_get_video_buffer(outlink, outlink->w, outlink->h);
176  if (!out) {
177  err = AVERROR(ENOMEM);
178  goto fail;
179  }
180 
181  if (!s->initialized)
182  RET(init_filter(ctx, in));
183 
184  RET(ff_vk_filter_process_simple(&s->vkctx, &s->e, &s->shd, out, in,
185  s->sampler, &s->opts, sizeof(s->opts)));
186 
187  err = av_frame_copy_props(out, in);
188  if (err < 0)
189  goto fail;
190 
191  av_frame_free(&in);
192 
193  return ff_filter_frame(outlink, out);
194 
195 fail:
196  av_frame_free(&in);
197  av_frame_free(&out);
198  return err;
199 }
200 
202 {
204  FFVulkanContext *vkctx = &s->vkctx;
205  FFVulkanFunctions *vk = &vkctx->vkfn;
206 
207  ff_vk_exec_pool_free(vkctx, &s->e);
208  ff_vk_shader_free(vkctx, &s->shd);
209 
210  if (s->sampler)
211  vk->DestroySampler(vkctx->hwctx->act_dev, s->sampler,
212  vkctx->hwctx->alloc);
213 
214  ff_vk_uninit(&s->vkctx);
215 
216  s->initialized = 0;
217 }
218 
219 #define OFFSET(x) offsetof(ChromaticAberrationVulkanContext, x)
220 #define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM)
222  { "dist_x", "Set horizontal distortion amount", OFFSET(opts.dist[0]), AV_OPT_TYPE_FLOAT, {.dbl = 0.0f}, -10.0f, 10.0f, .flags = FLAGS },
223  { "dist_y", "Set vertical distortion amount", OFFSET(opts.dist[1]), AV_OPT_TYPE_FLOAT, {.dbl = 0.0f}, -10.0f, 10.0f, .flags = FLAGS },
224  { NULL },
225 };
226 
227 AVFILTER_DEFINE_CLASS(chromaber_vulkan);
228 
230  {
231  .name = "default",
232  .type = AVMEDIA_TYPE_VIDEO,
233  .filter_frame = &chromaber_vulkan_filter_frame,
234  .config_props = &ff_vk_filter_config_input,
235  },
236 };
237 
239  {
240  .name = "default",
241  .type = AVMEDIA_TYPE_VIDEO,
242  .config_props = &ff_vk_filter_config_output,
243  },
244 };
245 
247  .name = "chromaber_vulkan",
248  .description = NULL_IF_CONFIG_SMALL("Offset chroma of input video (chromatic aberration)"),
249  .priv_size = sizeof(ChromaticAberrationVulkanContext),
255  .priv_class = &chromaber_vulkan_class,
256  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
257  .flags = AVFILTER_FLAG_HWDEVICE,
258 };
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:116
r
const char * r
Definition: vf_curves.c:127
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_vk_shader_free
void ff_vk_shader_free(FFVulkanContext *s, FFVulkanShader *shd)
Free a shader.
Definition: vulkan.c:2413
ff_vk_shader_init
int ff_vk_shader_init(FFVulkanContext *s, FFVulkanShader *shd, const char *name, VkPipelineStageFlags stage, const char *extensions[], int nb_extensions, int lg_x, int lg_y, int lg_z, uint32_t required_subgroup_size)
Initialize a shader object, with a specific set of extensions, type+bind, local group size,...
Definition: vulkan.c:1564
out
FILE * out
Definition: movenc.c:55
ChromaticAberrationVulkanContext::dist
float dist[2]
Definition: vf_chromaber_vulkan.c:40
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1062
RET
#define RET(x)
Definition: vulkan.h:67
ff_vk_filter_process_simple
int ff_vk_filter_process_simple(FFVulkanContext *vkctx, FFVkExecPool *e, FFVulkanShader *shd, AVFrame *out_f, AVFrame *in_f, VkSampler sampler, void *push_src, size_t push_size)
Submit a compute shader with a zero/one input and single out for execution.
Definition: vulkan_filter.c:242
ff_vk_qf_init
int ff_vk_qf_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, VkQueueFlagBits dev_family)
Chooses a QF and loads it into a context.
Definition: vulkan.c:228
ChromaticAberrationVulkanContext::shd
FFVulkanShader shd
Definition: vf_chromaber_vulkan.c:35
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:162
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: filters.h:262
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
chromaber_vulkan_uninit
static void chromaber_vulkan_uninit(AVFilterContext *avctx)
Definition: vf_chromaber_vulkan.c:201
ff_vk_filter_init
int ff_vk_filter_init(AVFilterContext *avctx)
General lavfi IO functions.
Definition: vulkan_filter.c:233
OFFSET
#define OFFSET(x)
Definition: vf_chromaber_vulkan.c:219
AVOption
AVOption.
Definition: opt.h:429
b
#define b
Definition: input.c:41
ff_vk_uninit
void ff_vk_uninit(FFVulkanContext *s)
Frees main context.
Definition: vulkan.c:2452
FFVkSPIRVCompiler::uninit
void(* uninit)(struct FFVkSPIRVCompiler **ctx)
Definition: vulkan_spirv.h:32
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:205
video.h
AV_PIX_FMT_VULKAN
@ AV_PIX_FMT_VULKAN
Vulkan hardware images.
Definition: pixfmt.h:379
av_pix_fmt_count_planes
int av_pix_fmt_count_planes(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:3065
ChromaticAberrationVulkanContext::sampler
VkSampler sampler
Definition: vf_chromaber_vulkan.c:36
ChromaticAberrationVulkanContext::opts
struct ChromaticAberrationVulkanContext::@333 opts
AVVulkanDeviceContext::alloc
const VkAllocationCallbacks * alloc
Custom memory allocator, else NULL.
Definition: hwcontext_vulkan.h:63
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:472
fail
#define fail()
Definition: checkasm.h:188
vulkan_filter.h
ff_vk_shader_register_exec
int ff_vk_shader_register_exec(FFVulkanContext *s, FFVkExecPool *pool, FFVulkanShader *shd)
Register a shader with an exec pool.
Definition: vulkan.c:2053
ChromaticAberrationVulkanContext::e
FFVkExecPool e
Definition: vf_chromaber_vulkan.c:33
ff_vk_shader_add_descriptor_set
int ff_vk_shader_add_descriptor_set(FFVulkanContext *s, FFVulkanShader *shd, FFVulkanDescriptorSetBinding *desc, int nb, int singular, int print_to_shader_only)
Add descriptor to a shader.
Definition: vulkan.c:1928
ChromaticAberrationVulkanContext
Definition: vf_chromaber_vulkan.c:29
AVFilterPad
A filter pad used for either input or output.
Definition: filters.h:38
C
s EdgeDetect Foobar g libavfilter vf_edgedetect c libavfilter vf_foobar c edit libavfilter and add an entry for foobar following the pattern of the other filters edit libavfilter allfilters and add an entry for foobar following the pattern of the other filters configure make j< whatever > ffmpeg ffmpeg i you should get a foobar png with Lena edge detected That s your new playground is ready Some little details about what s going which in turn will define variables for the build system and the C
Definition: writing_filters.txt:58
GLSLC
#define GLSLC(N, S)
Definition: vulkan.h:44
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
av_cold
#define av_cold
Definition: attributes.h:90
main
int main
Definition: dovi_rpuenc.c:37
FLAGS
#define FLAGS
Definition: vf_chromaber_vulkan.c:220
ff_vf_chromaber_vulkan
const AVFilter ff_vf_chromaber_vulkan
Definition: vf_chromaber_vulkan.c:246
s
#define s(width, name)
Definition: cbs_vp9.c:198
g
const char * g
Definition: vf_curves.c:128
filters.h
FF_VK_REP_FLOAT
@ FF_VK_REP_FLOAT
Definition: vulkan.h:367
chromaber_vulkan_options
static const AVOption chromaber_vulkan_options[]
Definition: vf_chromaber_vulkan.c:221
ctx
AVFormatContext * ctx
Definition: movenc.c:49
GLSLD
#define GLSLD(D)
Definition: vulkan.h:59
ff_vk_exec_pool_free
void ff_vk_exec_pool_free(FFVulkanContext *s, FFVkExecPool *pool)
Definition: vulkan.c:238
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: filters.h:263
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
opts
AVDictionary * opts
Definition: movenc.c:51
ff_vk_shader_rep_fmt
const char * ff_vk_shader_rep_fmt(enum AVPixelFormat pix_fmt, enum FFVkShaderRepFormat rep_fmt)
Definition: vulkan.c:1290
NULL
#define NULL
Definition: coverity.c:32
av_frame_copy_props
int av_frame_copy_props(AVFrame *dst, const AVFrame *src)
Copy only "metadata" fields from src to dst.
Definition: frame.c:713
DUP_SAMPLER
#define DUP_SAMPLER(x)
Definition: vulkan.h:73
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(chromaber_vulkan)
ff_vk_filter_config_output
int ff_vk_filter_config_output(AVFilterLink *outlink)
Definition: vulkan_filter.c:209
ff_vk_exec_pool_init
int ff_vk_exec_pool_init(FFVulkanContext *s, FFVkQueueFamilyCtx *qf, FFVkExecPool *pool, int nb_contexts, int nb_queries, VkQueryType query_type, int query_64bit, const void *query_create_pnext)
Allocates/frees an execution pool.
Definition: vulkan.c:300
FFVulkanContext
Definition: vulkan.h:263
chromaber_vulkan_outputs
static const AVFilterPad chromaber_vulkan_outputs[]
Definition: vf_chromaber_vulkan.c:238
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: filters.h:206
init_filter
static av_cold int init_filter(AVFilterContext *ctx, AVFrame *in)
Definition: vf_chromaber_vulkan.c:68
f
f
Definition: af_crystalizer.c:122
init
int(* init)(AVBSFContext *ctx)
Definition: dts2pts.c:368
FFVulkanDescriptorSetBinding
Definition: vulkan.h:75
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:94
AVFILTER_FLAG_HWDEVICE
#define AVFILTER_FLAG_HWDEVICE
The filter can create hardware frames using AVFilterContext.hw_device_ctx.
Definition: avfilter.h:173
size
int size
Definition: twinvq_data.h:10344
FFVkQueueFamilyCtx
Definition: vulkan.h:102
FFVulkanShader
Definition: vulkan.h:179
chromaber_vulkan_filter_frame
static int chromaber_vulkan_filter_frame(AVFilterLink *link, AVFrame *in)
Definition: vf_chromaber_vulkan.c:168
FFVkSPIRVCompiler::compile_shader
int(* compile_shader)(FFVulkanContext *s, struct FFVkSPIRVCompiler *ctx, FFVulkanShader *shd, uint8_t **data, size_t *size, const char *entrypoint, void **opaque)
Definition: vulkan_spirv.h:28
a
The reader does not expect b to be semantically here and if the code is changed by maybe adding a a division or other the signedness will almost certainly be mistaken To avoid this confusion a new type was SUINT is the C unsigned type but it holds a signed int to use the same example SUINT a
Definition: undefined.txt:41
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
FFVkSPIRVCompiler
Definition: vulkan_spirv.h:26
AV_OPT_TYPE_FLOAT
@ AV_OPT_TYPE_FLOAT
Underlying C type is float.
Definition: opt.h:271
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
ChromaticAberrationVulkanContext::initialized
int initialized
Definition: vf_chromaber_vulkan.c:32
uninit
static void uninit(AVBSFContext *ctx)
Definition: pcm_rechunk.c:68
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
ff_vk_shader_link
int ff_vk_shader_link(FFVulkanContext *s, FFVulkanShader *shd, uint8_t *spirv, size_t spirv_len, const char *entrypoint)
Link a shader into an executable.
Definition: vulkan.c:1853
vulkan_spirv.h
AVFilterPad::name
const char * name
Pad name.
Definition: filters.h:44
FFVkSPIRVCompiler::free_shader
void(* free_shader)(struct FFVkSPIRVCompiler *ctx, void **opaque)
Definition: vulkan_spirv.h:31
ChromaticAberrationVulkanContext::vkctx
FFVulkanContext vkctx
Definition: vf_chromaber_vulkan.c:30
AVFilter
Filter definition.
Definition: avfilter.h:201
FFVulkanContext::vkfn
FFVulkanFunctions vkfn
Definition: vulkan.h:267
FFVkExecPool
Definition: vulkan.h:241
pos
unsigned int pos
Definition: spdifenc.c:414
ff_vk_shader_add_push_const
int ff_vk_shader_add_push_const(FFVulkanShader *shd, int offset, int size, VkShaderStageFlagBits stage)
Add/update push constants for execution.
Definition: vulkan.c:1223
random_seed.h
GLSLF
#define GLSLF(N, S,...)
Definition: vulkan.h:54
AVFilterContext
An instance of a filter.
Definition: avfilter.h:457
desc
const char * desc
Definition: libsvtav1.c:79
ff_vk_filter_config_input
int ff_vk_filter_config_input(AVFilterLink *inlink)
Definition: vulkan_filter.c:176
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
FFVulkanContext::hwctx
AVVulkanDeviceContext * hwctx
Definition: vulkan.h:291
AVVulkanDeviceContext::act_dev
VkDevice act_dev
Active device.
Definition: hwcontext_vulkan.h:84
ff_vk_init_sampler
int ff_vk_init_sampler(FFVulkanContext *s, VkSampler *sampler, int unnorm_coords, VkFilter filt)
Create a sampler.
Definition: vulkan.c:1244
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
distort_chroma_kernel
static const char distort_chroma_kernel[]
Definition: vf_chromaber_vulkan.c:44
chromaber_vulkan_inputs
static const AVFilterPad chromaber_vulkan_inputs[]
Definition: vf_chromaber_vulkan.c:229
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: filters.h:252
FFVulkanFunctions
Definition: vulkan_functions.h:263
planes
static const struct @458 planes[]
ChromaticAberrationVulkanContext::qf
FFVkQueueFamilyCtx qf
Definition: vf_chromaber_vulkan.c:34