FFmpeg
vf_coreimage.m
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2016 Thilo Borgmann
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Video processing based on Apple's CoreImage API
24  */
25 
26 #import <CoreImage/CoreImage.h>
27 #import <AppKit/AppKit.h>
28 
29 #include "avfilter.h"
30 #include "formats.h"
31 #include "internal.h"
32 #include "video.h"
33 #include "libavutil/internal.h"
34 #include "libavutil/opt.h"
35 #include "libavutil/pixdesc.h"
36 
37 typedef struct CoreImageContext {
38  const AVClass *class;
39 
40  int is_video_source; ///< filter is used as video source
41 
42  int w, h; ///< video size
43  AVRational sar; ///< sample aspect ratio
44  AVRational frame_rate; ///< video frame rate
45  AVRational time_base; ///< stream time base
46  int64_t duration; ///< duration expressed in microseconds
47  int64_t pts; ///< increasing presentation time stamp
48  AVFrame *picref; ///< cached reference containing the painted picture
49 
50  CFTypeRef glctx; ///< OpenGL context
51  CGContextRef cgctx; ///< Bitmap context for image copy
52  CFTypeRef input_image; ///< Input image container for passing into Core Image API
53  CGColorSpaceRef color_space; ///< Common color space for input image and cgcontext
54  int bits_per_component; ///< Shared bpc for input-output operation
55 
56  char *filter_string; ///< The complete user provided filter definition
57  CFTypeRef *filters; ///< CIFilter object for all requested filters
58  int num_filters; ///< Amount of filters in *filters
59 
60  char *output_rect; ///< Rectangle to be filled with filter intput
61  int list_filters; ///< Option used to list all available filters including generators
62  int list_generators; ///< Option used to list all available generators
64 
66 {
67  CoreImageContext *ctx = link->src->priv;
68 
69  link->w = ctx->w;
70  link->h = ctx->h;
72  link->frame_rate = ctx->frame_rate;
73  link->time_base = ctx->time_base;
74 
76  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
77 
78  return 0;
79 }
80 
81 /** Determine image properties from input link of filter chain.
82  */
84 {
85  CoreImageContext *ctx = link->dst->priv;
87  ctx->bits_per_component = av_get_bits_per_pixel(desc) / desc->nb_components;
88 
89  return 0;
90 }
91 
92 /** Print a list of all available filters including options and respective value ranges and defaults.
93  */
95 {
96  // querying filters and attributes
97  NSArray *filter_categories = nil;
98 
99  if (ctx->list_generators && !ctx->list_filters) {
100  filter_categories = [NSArray arrayWithObjects:kCICategoryGenerator, nil];
101  }
102 
103  NSArray *filter_names = [CIFilter filterNamesInCategories:filter_categories];
104  NSEnumerator *filters = [filter_names objectEnumerator];
105 
106  NSString *filter_name;
107  while (filter_name = [filters nextObject]) {
108  av_log(ctx, AV_LOG_INFO, "Filter: %s\n", [filter_name UTF8String]);
109  NSString *input;
110 
111  CIFilter *filter = [CIFilter filterWithName:filter_name];
112  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
113  NSArray *filter_inputs = [filter inputKeys]; // <nsstring>
114 
115  for (input in filter_inputs) {
116  NSDictionary *input_attribs = [filter_attribs valueForKey:input];
117  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
118  if ([input_class isEqualToString:@"NSNumber"]) {
119  NSNumber *value_default = [input_attribs valueForKey:kCIAttributeDefault];
120  NSNumber *value_min = [input_attribs valueForKey:kCIAttributeSliderMin];
121  NSNumber *value_max = [input_attribs valueForKey:kCIAttributeSliderMax];
122 
123  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\t[%s %s][%s]\n",
124  [input UTF8String],
125  [input_class UTF8String],
126  [[value_min stringValue] UTF8String],
127  [[value_max stringValue] UTF8String],
128  [[value_default stringValue] UTF8String]);
129  } else {
130  av_log(ctx, AV_LOG_INFO, "\tOption: %s\t[%s]\n",
131  [input UTF8String],
132  [input_class UTF8String]);
133  }
134  }
135  }
136 }
137 
139 {
140  int i;
141 
142  // (re-)initialize input image
143  const CGSize frame_size = {
144  frame->width,
145  frame->height
146  };
147 
148  NSData *data = [NSData dataWithBytesNoCopy:frame->data[0]
149  length:frame->height*frame->linesize[0]
150  freeWhenDone:NO];
151 
152  CIImage *ret = [(__bridge CIImage*)ctx->input_image initWithBitmapData:data
153  bytesPerRow:frame->linesize[0]
154  size:frame_size
155  format:kCIFormatARGB8
156  colorSpace:ctx->color_space]; //kCGColorSpaceGenericRGB
157  if (!ret) {
158  av_log(ctx, AV_LOG_ERROR, "Input image could not be initialized.\n");
159  return AVERROR_EXTERNAL;
160  }
161 
162  CIFilter *filter = NULL;
163  CIImage *filter_input = (__bridge CIImage*)ctx->input_image;
164  CIImage *filter_output = NULL;
165 
166  // successively apply all filters
167  for (i = 0; i < ctx->num_filters; i++) {
168  if (i) {
169  // set filter input to previous filter output
170  filter_input = [(__bridge CIImage*)ctx->filters[i-1] valueForKey:kCIOutputImageKey];
171  CGRect out_rect = [filter_input extent];
172  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
173  // do not keep padded image regions after filtering
174  out_rect.origin.x = 0.0f;
175  out_rect.origin.y = 0.0f;
176  out_rect.size.width = frame->width;
177  out_rect.size.height = frame->height;
178  }
179  filter_input = [filter_input imageByCroppingToRect:out_rect];
180  }
181 
182  filter = (__bridge CIFilter*)ctx->filters[i];
183 
184  // do not set input image for the first filter if used as video source
185  if (!ctx->is_video_source || i) {
186  @try {
187  [filter setValue:filter_input forKey:kCIInputImageKey];
188  } @catch (NSException *exception) {
189  if (![[exception name] isEqualToString:NSUndefinedKeyException]) {
190  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
191  return AVERROR_EXTERNAL;
192  } else {
193  av_log(ctx, AV_LOG_WARNING, "Selected filter does not accept an input image.\n");
194  }
195  }
196  }
197  }
198 
199  // get output of last filter
200  filter_output = [filter valueForKey:kCIOutputImageKey];
201 
202  if (!filter_output) {
203  av_log(ctx, AV_LOG_ERROR, "Filter output not available.\n");
204  return AVERROR_EXTERNAL;
205  }
206 
207  // do not keep padded image regions after filtering
208  CGRect out_rect = [filter_output extent];
209  if (out_rect.size.width > frame->width || out_rect.size.height > frame->height) {
210  av_log(ctx, AV_LOG_DEBUG, "Cropping output image.\n");
211  out_rect.origin.x = 0.0f;
212  out_rect.origin.y = 0.0f;
213  out_rect.size.width = frame->width;
214  out_rect.size.height = frame->height;
215  }
216 
217  CGImageRef out = [(__bridge CIContext*)ctx->glctx createCGImage:filter_output
218  fromRect:out_rect];
219 
220  if (!out) {
221  av_log(ctx, AV_LOG_ERROR, "Cannot create valid output image.\n");
222  }
223 
224  // create bitmap context on the fly for rendering into current frame->data[]
225  if (ctx->cgctx) {
226  CGContextRelease(ctx->cgctx);
227  ctx->cgctx = NULL;
228  }
229  size_t out_width = CGImageGetWidth(out);
230  size_t out_height = CGImageGetHeight(out);
231 
232  if (out_width > frame->width || out_height > frame->height) { // this might result in segfault
233  av_log(ctx, AV_LOG_WARNING, "Output image has unexpected size: %lux%lu (expected: %ix%i). This may crash...\n",
234  out_width, out_height, frame->width, frame->height);
235  }
236  ctx->cgctx = CGBitmapContextCreate(frame->data[0],
237  frame->width,
238  frame->height,
239  ctx->bits_per_component,
240  frame->linesize[0],
241  ctx->color_space,
242  (uint32_t)kCGImageAlphaPremultipliedFirst); // ARGB
243  if (!ctx->cgctx) {
244  av_log(ctx, AV_LOG_ERROR, "CGBitmap context cannot be created.\n");
245  return AVERROR_EXTERNAL;
246  }
247 
248  // copy ("draw") the output image into the frame data
249  CGRect rect = {{0,0},{frame->width, frame->height}};
250  if (ctx->output_rect) {
251  @try {
252  NSString *tmp_string = [NSString stringWithUTF8String:ctx->output_rect];
253  NSRect tmp = NSRectFromString(tmp_string);
254  rect = NSRectToCGRect(tmp);
255  } @catch (NSException *exception) {
256  av_log(ctx, AV_LOG_ERROR, "An error occurred: %s.", [exception.reason UTF8String]);
257  return AVERROR_EXTERNAL;
258  }
259  if (rect.size.width == 0.0f) {
260  av_log(ctx, AV_LOG_WARNING, "Width of output rect is zero.\n");
261  }
262  if (rect.size.height == 0.0f) {
263  av_log(ctx, AV_LOG_WARNING, "Height of output rect is zero.\n");
264  }
265  }
266 
267  CGContextDrawImage(ctx->cgctx, rect, out);
268 
269  return ff_filter_frame(link, frame);
270 }
271 
272 /** Apply all valid filters successively to the input image.
273  * The final output image is copied from the GPU by "drawing" using a bitmap context.
274  */
276 {
277  return apply_filter(link->dst->priv, link->dst->outputs[0], frame);
278 }
279 
281 {
282  CoreImageContext *ctx = link->src->priv;
283  AVFrame *frame;
284 
285  if (ctx->duration >= 0 &&
286  av_rescale_q(ctx->pts, ctx->time_base, AV_TIME_BASE_Q) >= ctx->duration) {
287  return AVERROR_EOF;
288  }
289 
290  if (!ctx->picref) {
291  ctx->picref = ff_get_video_buffer(link, ctx->w, ctx->h);
292  if (!ctx->picref) {
293  return AVERROR(ENOMEM);
294  }
295  }
296 
297  frame = av_frame_clone(ctx->picref);
298  if (!frame) {
299  return AVERROR(ENOMEM);
300  }
301 
302  frame->pts = ctx->pts;
303  frame->duration = 1;
304  frame->key_frame = 1;
305  frame->interlaced_frame = 0;
306  frame->pict_type = AV_PICTURE_TYPE_I;
307  frame->sample_aspect_ratio = ctx->sar;
308 
309  ctx->pts++;
310 
311  return apply_filter(ctx, link, frame);
312 }
313 
314 /** Set an option of the given filter to the provided key-value pair.
315  */
316 static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
317 {
318  NSString *input_key = [NSString stringWithUTF8String:key];
319  NSString *input_val = [NSString stringWithUTF8String:value];
320 
321  NSDictionary *filter_attribs = [filter attributes]; // <nsstring, id>
322  NSDictionary *input_attribs = [filter_attribs valueForKey:input_key];
323 
324  NSString *input_class = [input_attribs valueForKey:kCIAttributeClass];
325  NSString *input_type = [input_attribs valueForKey:kCIAttributeType];
326 
327  if (!input_attribs) {
328  av_log(ctx, AV_LOG_WARNING, "Skipping unknown option: \"%s\".\n",
329  [input_key UTF8String]); // [[filter name] UTF8String]) not currently defined...
330  return;
331  }
332 
333  av_log(ctx, AV_LOG_DEBUG, "key: %s, val: %s, #attribs: %lu, class: %s, type: %s\n",
334  [input_key UTF8String],
335  [input_val UTF8String],
336  input_attribs ? (unsigned long)[input_attribs count] : -1,
337  [input_class UTF8String],
338  [input_type UTF8String]);
339 
340  if ([input_class isEqualToString:@"NSNumber"]) {
341  float input = input_val.floatValue;
342  NSNumber *max_value = [input_attribs valueForKey:kCIAttributeSliderMax];
343  NSNumber *min_value = [input_attribs valueForKey:kCIAttributeSliderMin];
344  NSNumber *used_value = nil;
345 
346 #define CLAMP_WARNING do { \
347 av_log(ctx, AV_LOG_WARNING, "Value of \"%f\" for option \"%s\" is out of range [%f %f], clamping to \"%f\".\n", \
348  input, \
349  [input_key UTF8String], \
350  min_value.floatValue, \
351  max_value.floatValue, \
352  used_value.floatValue); \
353 } while(0)
354  if (input > max_value.floatValue) {
355  used_value = max_value;
357  } else if (input < min_value.floatValue) {
358  used_value = min_value;
360  } else {
361  used_value = [NSNumber numberWithFloat:input];
362  }
363 
364  [filter setValue:used_value forKey:input_key];
365  } else if ([input_class isEqualToString:@"CIVector"]) {
366  CIVector *input = [CIVector vectorWithString:input_val];
367 
368  if (!input) {
369  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIVctor description: \"%s\".\n",
370  [input_val UTF8String]);
371  return;
372  }
373 
374  [filter setValue:input forKey:input_key];
375  } else if ([input_class isEqualToString:@"CIColor"]) {
376  CIColor *input = [CIColor colorWithString:input_val];
377 
378  if (!input) {
379  av_log(ctx, AV_LOG_WARNING, "Skipping invalid CIColor description: \"%s\".\n",
380  [input_val UTF8String]);
381  return;
382  }
383 
384  [filter setValue:input forKey:input_key];
385  } else if ([input_class isEqualToString:@"NSString"]) { // set display name as string with latin1 encoding
386  [filter setValue:input_val forKey:input_key];
387  } else if ([input_class isEqualToString:@"NSData"]) { // set display name as string with latin1 encoding
388  NSData *input = [NSData dataWithBytes:(const void*)[input_val cStringUsingEncoding:NSISOLatin1StringEncoding]
389  length:[input_val lengthOfBytesUsingEncoding:NSISOLatin1StringEncoding]];
390 
391  if (!input) {
392  av_log(ctx, AV_LOG_WARNING, "Skipping invalid NSData description: \"%s\".\n",
393  [input_val UTF8String]);
394  return;
395  }
396 
397  [filter setValue:input forKey:input_key];
398  } else {
399  av_log(ctx, AV_LOG_WARNING, "Skipping unsupported option class: \"%s\".\n",
400  [input_class UTF8String]);
401  avpriv_report_missing_feature(ctx, "Handling of some option classes");
402  return;
403  }
404 }
405 
406 /** Create a filter object by a given name and set all options to defaults.
407  * Overwrite any option given by the user to the provided value in filter_options.
408  */
409 static CIFilter* create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
410 {
411  // create filter object
412  CIFilter *filter = [CIFilter filterWithName:[NSString stringWithUTF8String:filter_name]];
413 
414  // set default options
415  [filter setDefaults];
416 
417  // set user options
418  if (filter_options) {
419  const AVDictionaryEntry *o = NULL;
420  while ((o = av_dict_iterate(filter_options, o))) {
421  set_option(ctx, filter, o->key, o->value);
422  }
423  }
424 
425  return filter;
426 }
427 
428 static av_cold int init(AVFilterContext *fctx)
429 {
430  CoreImageContext *ctx = fctx->priv;
431  AVDictionary *filter_dict = NULL;
432  const AVDictionaryEntry *f = NULL;
433  const AVDictionaryEntry *o = NULL;
434  int ret;
435  int i;
436 
437  if (ctx->list_filters || ctx->list_generators) {
438  list_filters(ctx);
439  return AVERROR_EXIT;
440  }
441 
442  if (ctx->filter_string) {
443  // parse filter string (filter=name@opt=val@opt2=val2#name2@opt3=val3) for filters separated by #
444  av_log(ctx, AV_LOG_DEBUG, "Filter_string: %s\n", ctx->filter_string);
445  ret = av_dict_parse_string(&filter_dict, ctx->filter_string, "@", "#", AV_DICT_MULTIKEY); // parse filter_name:all_filter_options
446  if (ret) {
447  av_dict_free(&filter_dict);
448  av_log(ctx, AV_LOG_ERROR, "Parsing of filters failed.\n");
449  return AVERROR(EIO);
450  }
451  ctx->num_filters = av_dict_count(filter_dict);
452  av_log(ctx, AV_LOG_DEBUG, "Filter count: %i\n", ctx->num_filters);
453 
454  // allocate CIFilter array
455  ctx->filters = av_calloc(ctx->num_filters, sizeof(CIFilter*));
456  if (!ctx->filters) {
457  av_log(ctx, AV_LOG_ERROR, "Could not allocate filter array.\n");
458  return AVERROR(ENOMEM);
459  }
460 
461  // parse filters for option key-value pairs (opt=val@opt2=val2) separated by @
462  i = 0;
463  while ((f = av_dict_iterate(filter_dict, f))) {
464  AVDictionary *filter_options = NULL;
465 
466  if (strncmp(f->value, "default", 7)) { // not default
467  ret = av_dict_parse_string(&filter_options, f->value, "=", "@", 0); // parse option_name:option_value
468  if (ret) {
469  av_dict_free(&filter_options);
470  av_log(ctx, AV_LOG_ERROR, "Parsing of filter options for \"%s\" failed.\n", f->key);
471  return AVERROR(EIO);
472  }
473  }
474 
475  if (av_log_get_level() >= AV_LOG_DEBUG) {
476  av_log(ctx, AV_LOG_DEBUG, "Creating filter %i: \"%s\":\n", i, f->key);
477  if (!filter_options) {
478  av_log(ctx, AV_LOG_DEBUG, "\tusing default options\n");
479  } else {
480  while ((o = av_dict_iterate(filter_options, o))) {
481  av_log(ctx, AV_LOG_DEBUG, "\t%s: %s\n", o->key, o->value);
482  }
483  }
484  }
485 
486  ctx->filters[i] = CFBridgingRetain(create_filter(ctx, f->key, filter_options));
487  if (!ctx->filters[i]) {
488  av_log(ctx, AV_LOG_ERROR, "Could not create filter \"%s\".\n", f->key);
489  return AVERROR(EINVAL);
490  }
491 
492  i++;
493  }
494  } else {
495  av_log(ctx, AV_LOG_ERROR, "No filters specified.\n");
496  return AVERROR(EINVAL);
497  }
498 
499  // create GPU context on OSX
500  const NSOpenGLPixelFormatAttribute attr[] = {
501  NSOpenGLPFAAccelerated,
502  NSOpenGLPFANoRecovery,
503  NSOpenGLPFAColorSize, 32,
504  0
505  };
506 
507  NSOpenGLPixelFormat *pixel_format = [[NSOpenGLPixelFormat alloc] initWithAttributes:(void *)&attr];
508  ctx->color_space = CGColorSpaceCreateWithName(kCGColorSpaceGenericRGB);
509  ctx->glctx = CFBridgingRetain([CIContext contextWithCGLContext:CGLGetCurrentContext()
510  pixelFormat:[pixel_format CGLPixelFormatObj]
511  colorSpace:ctx->color_space
512  options:nil]);
513 
514  if (!ctx->glctx) {
515  av_log(ctx, AV_LOG_ERROR, "CIContext not created.\n");
516  return AVERROR_EXTERNAL;
517  }
518 
519  // Creating an empty input image as input container for the context
520  ctx->input_image = CFBridgingRetain([CIImage emptyImage]);
521 
522  return 0;
523 }
524 
526 {
527  CoreImageContext *ctx = fctx->priv;
528 
529  ctx->is_video_source = 1;
530  ctx->time_base = av_inv_q(ctx->frame_rate);
531  ctx->pts = 0;
532 
533  return init(fctx);
534 }
535 
536 static av_cold void uninit(AVFilterContext *fctx)
537 {
538 #define SafeCFRelease(ptr) do { \
539  if (ptr) { \
540  CFRelease(ptr); \
541  ptr = NULL; \
542  } \
543 } while (0)
544 
545  CoreImageContext *ctx = fctx->priv;
546 
547  SafeCFRelease(ctx->glctx);
548  SafeCFRelease(ctx->cgctx);
549  SafeCFRelease(ctx->color_space);
550  SafeCFRelease(ctx->input_image);
551 
552  if (ctx->filters) {
553  for (int i = 0; i < ctx->num_filters; i++) {
554  SafeCFRelease(ctx->filters[i]);
555  }
556  av_freep(&ctx->filters);
557  }
558 
559  av_frame_free(&ctx->picref);
560 }
561 
563  {
564  .name = "default",
565  .type = AVMEDIA_TYPE_VIDEO,
566  .filter_frame = filter_frame,
567  .config_props = config_input,
568  },
569 };
570 
572  {
573  .name = "default",
574  .type = AVMEDIA_TYPE_VIDEO,
575  },
576 };
577 
579  {
580  .name = "default",
581  .type = AVMEDIA_TYPE_VIDEO,
582  .request_frame = request_frame,
583  .config_props = config_output,
584  },
585 };
586 
587 #define OFFSET(x) offsetof(CoreImageContext, x)
588 #define FLAGS AV_OPT_FLAG_VIDEO_PARAM|AV_OPT_FLAG_FILTERING_PARAM
589 
590 #define GENERATOR_OPTIONS \
591  {"size", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
592  {"s", "set video size", OFFSET(w), AV_OPT_TYPE_IMAGE_SIZE, {.str = "320x240"}, 0, 0, FLAGS}, \
593  {"rate", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
594  {"r", "set video rate", OFFSET(frame_rate), AV_OPT_TYPE_VIDEO_RATE, {.str = "25"}, 0, INT_MAX, FLAGS}, \
595  {"duration", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
596  {"d", "set video duration", OFFSET(duration), AV_OPT_TYPE_DURATION, {.i64 = -1}, -1, INT64_MAX, FLAGS}, \
597  {"sar", "set video sample aspect ratio", OFFSET(sar), AV_OPT_TYPE_RATIONAL, {.dbl = 1}, 0, INT_MAX, FLAGS},
598 
599 #define FILTER_OPTIONS \
600  {"list_filters", "list available filters", OFFSET(list_filters), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
601  {"list_generators", "list available generators", OFFSET(list_generators), AV_OPT_TYPE_BOOL, {.i64 = 0}, 0, 1, .flags = FLAGS}, \
602  {"filter", "names and options of filters to apply", OFFSET(filter_string), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS}, \
603  {"output_rect", "output rectangle within output image", OFFSET(output_rect), AV_OPT_TYPE_STRING, {.str = NULL}, .flags = FLAGS},
604 
605 
606 // definitions for coreimage video filter
607 static const AVOption coreimage_options[] = {
609  { NULL }
610 };
611 
612 AVFILTER_DEFINE_CLASS(coreimage);
613 
615  .name = "coreimage",
616  .description = NULL_IF_CONFIG_SMALL("Video filtering using CoreImage API."),
617  .init = init,
618  .uninit = uninit,
619  .priv_size = sizeof(CoreImageContext),
620  .priv_class = &coreimage_class,
624 };
625 
626 // definitions for coreimagesrc video source
627 static const AVOption coreimagesrc_options[] = {
630  { NULL }
631 };
632 
633 AVFILTER_DEFINE_CLASS(coreimagesrc);
634 
636  .name = "coreimagesrc",
637  .description = NULL_IF_CONFIG_SMALL("Video source using image generators of CoreImage API."),
638  .init = init_src,
639  .uninit = uninit,
640  .priv_size = sizeof(CoreImageContext),
641  .priv_class = &coreimagesrc_class,
642  .inputs = NULL,
645 };
ff_vsrc_coreimagesrc
const AVFilter ff_vsrc_coreimagesrc
Definition: vf_coreimage.m:635
ff_get_video_buffer
AVFrame * ff_get_video_buffer(AVFilterLink *link, int w, int h)
Request a picture buffer with a specific set of permissions.
Definition: video.c:101
CoreImageContext::glctx
CFTypeRef glctx
OpenGL context.
Definition: vf_coreimage.m:50
CoreImageContext::w
int w
Definition: vf_coreimage.m:42
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
name
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option name
Definition: writing_filters.txt:88
CoreImageContext::duration
int64_t duration
duration expressed in microseconds
Definition: vf_coreimage.m:46
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
out
FILE * out
Definition: movenc.c:54
CoreImageContext::cgctx
CGContextRef cgctx
Bitmap context for image copy.
Definition: vf_coreimage.m:51
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1009
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2888
AVERROR_EOF
#define AVERROR_EOF
End of file.
Definition: error.h:57
av_dict_count
int av_dict_count(const AVDictionary *m)
Get number of entries in dictionary.
Definition: dict.c:37
rect
Definition: f_ebur128.c:76
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(coreimage)
AV_TIME_BASE_Q
#define AV_TIME_BASE_Q
Internal time base represented as fractional value.
Definition: avutil.h:260
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:116
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:325
tmp
static uint8_t tmp[11]
Definition: aes_ctr.c:28
pixdesc.h
AVOption
AVOption.
Definition: opt.h:251
create_filter
static CIFilter * create_filter(CoreImageContext *ctx, const char *filter_name, AVDictionary *filter_options)
Create a filter object by a given name and set all options to defaults.
Definition: vf_coreimage.m:409
data
const char data[16]
Definition: mxf.c:146
CoreImageContext::color_space
CGColorSpaceRef color_space
Common color space for input image and cgcontext.
Definition: vf_coreimage.m:53
vf_coreimage_outputs
static const AVFilterPad vf_coreimage_outputs[]
Definition: vf_coreimage.m:571
av_get_bits_per_pixel
int av_get_bits_per_pixel(const AVPixFmtDescriptor *pixdesc)
Return the number of bits per pixel used by the pixel format described by pixdesc.
Definition: pixdesc.c:2840
filter
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce then the filter should push the output frames on the output link immediately As an exception to the previous rule if the input frame is enough to produce several output frames then the filter needs output only at least one per link The additional frames can be left buffered in the filter
Definition: filter_design.txt:228
AVDictionary
Definition: dict.c:32
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:175
coreimage_options
static const AVOption coreimage_options[]
Definition: vf_coreimage.m:607
video.h
CoreImageContext
Definition: vf_coreimage.m:37
formats.h
ff_vf_coreimage
const AVFilter ff_vf_coreimage
Definition: vf_coreimage.m:614
filter_frame
static int filter_frame(AVFilterLink *link, AVFrame *frame)
Apply all valid filters successively to the input image.
Definition: vf_coreimage.m:275
CLAMP_WARNING
#define CLAMP_WARNING
AVFilterContext::priv
void * priv
private data for use by the filter
Definition: avfilter.h:430
vsrc_coreimagesrc_outputs
static const AVFilterPad vsrc_coreimagesrc_outputs[]
Definition: vf_coreimage.m:578
request_frame
static int request_frame(AVFilterLink *link)
Definition: vf_coreimage.m:280
init
static av_cold int init(AVFilterContext *fctx)
Definition: vf_coreimage.m:428
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:49
config_output
static int config_output(AVFilterLink *link)
Definition: vf_coreimage.m:65
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
CoreImageContext::picref
AVFrame * picref
cached reference containing the painted picture
Definition: vf_coreimage.m:48
AVDictionaryEntry::key
char * key
Definition: dict.h:90
frame_size
int frame_size
Definition: mxfenc.c:2202
FILTER_OPTIONS
#define FILTER_OPTIONS
Definition: vf_coreimage.m:599
filters
#define filters(fmt, type, inverse, clp, inverset, clip, one, clip_fn, packed)
Definition: af_crystalizer.c:55
CoreImageContext::h
int h
video size
Definition: vf_coreimage.m:42
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
av_frame_clone
AVFrame * av_frame_clone(const AVFrame *src)
Create a new frame that references the same data as src.
Definition: frame.c:474
av_rescale_q
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
CoreImageContext::time_base
AVRational time_base
stream time base
Definition: vf_coreimage.m:45
key
const char * key
Definition: hwcontext_opencl.c:174
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:190
link
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a link
Definition: filter_design.txt:23
if
if(ret)
Definition: filter_design.txt:179
config_input
static int config_input(AVFilterLink *link)
Determine image properties from input link of filter chain.
Definition: vf_coreimage.m:83
av_log_get_level
int av_log_get_level(void)
Get the current log level.
Definition: log.c:437
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AV_DICT_MULTIKEY
#define AV_DICT_MULTIKEY
Allow to store several equal keys in the dictionary.
Definition: dict.h:84
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
CoreImageContext::filters
CFTypeRef * filters
CIFilter object for all requested filters.
Definition: vf_coreimage.m:57
CoreImageContext::bits_per_component
int bits_per_component
Shared bpc for input-output operation.
Definition: vf_coreimage.m:54
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
CoreImageContext::pts
int64_t pts
increasing presentation time stamp
Definition: vf_coreimage.m:47
options
const OptionDef options[]
f
f
Definition: af_crystalizer.c:122
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:115
CoreImageContext::sar
AVRational sar
sample aspect ratio
Definition: vf_coreimage.m:43
CoreImageContext::input_image
CFTypeRef input_image
Input image container for passing into Core Image API.
Definition: vf_coreimage.m:52
for
for(k=2;k<=8;++k)
Definition: h264pred_template.c:425
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
AVFrame::time_base
AVRational time_base
Time base for the timestamps in this frame.
Definition: frame.h:447
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
Definition: frame.h:412
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:223
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
internal.h
AV_PIX_FMT_ARGB
@ AV_PIX_FMT_ARGB
packed ARGB 8:8:8:8, 32bpp, ARGBARGB...
Definition: pixfmt.h:92
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:180
list_filters
static void list_filters(CoreImageContext *ctx)
Print a list of all available filters including options and respective value ranges and defaults.
Definition: vf_coreimage.m:94
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
coreimagesrc_options
static const AVOption coreimagesrc_options[]
Definition: vf_coreimage.m:627
internal.h
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
av_inv_q
static av_always_inline AVRational av_inv_q(AVRational q)
Invert a rational.
Definition: rational.h:159
CoreImageContext::filter_string
char * filter_string
The complete user provided filter definition.
Definition: vf_coreimage.m:56
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:55
set_option
static void set_option(CoreImageContext *ctx, CIFilter *filter, const char *key, const char *value)
Set an option of the given filter to the provided key-value pair.
Definition: vf_coreimage.m:316
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:272
CoreImageContext::frame_rate
AVRational frame_rate
video frame rate
Definition: vf_coreimage.m:44
AVFilter
Filter definition.
Definition: avfilter.h:171
ret
ret
Definition: filter_design.txt:187
CoreImageContext::output_rect
char * output_rect
Rectangle to be filled with filter intput.
Definition: vf_coreimage.m:60
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
AVFrame::sample_aspect_ratio
AVRational sample_aspect_ratio
Sample aspect ratio for the video frame, 0/1 if unknown/unspecified.
Definition: frame.h:427
CoreImageContext::num_filters
int num_filters
Amount of filters in *filters.
Definition: vf_coreimage.m:58
CoreImageContext::list_generators
int list_generators
Option used to list all available generators.
Definition: vf_coreimage.m:62
init_src
static av_cold int init_src(AVFilterContext *fctx)
Definition: vf_coreimage.m:525
avfilter.h
av_dict_parse_string
int av_dict_parse_string(AVDictionary **pm, const char *str, const char *key_val_sep, const char *pairs_sep, int flags)
Parse the key/value pairs list and add the parsed entries to a dictionary.
Definition: dict.c:200
CoreImageContext::is_video_source
int is_video_source
filter is used as video source
Definition: vf_coreimage.m:40
apply_filter
static int apply_filter(CoreImageContext *ctx, AVFilterLink *link, AVFrame *frame)
Definition: vf_coreimage.m:138
AVFormatContext::duration
int64_t duration
Duration of the stream, in AV_TIME_BASE fractional seconds.
Definition: avformat.h:1318
AVFilterContext
An instance of a filter.
Definition: avfilter.h:415
desc
const char * desc
Definition: libsvtav1.c:83
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
uninit
static av_cold void uninit(AVFilterContext *fctx)
Definition: vf_coreimage.m:536
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
AVDictionaryEntry
Definition: dict.h:89
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:191
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
vf_coreimage_inputs
static const AVFilterPad vf_coreimage_inputs[]
Definition: vf_coreimage.m:562
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
GENERATOR_OPTIONS
#define GENERATOR_OPTIONS
Definition: vf_coreimage.m:590
AVERROR_EXIT
#define AVERROR_EXIT
Immediate exit was requested; the called function should not be restarted.
Definition: error.h:58
SafeCFRelease
#define SafeCFRelease(ptr)
AVDictionaryEntry::value
char * value
Definition: dict.h:91
av_dict_iterate
const AVDictionaryEntry * av_dict_iterate(const AVDictionary *m, const AVDictionaryEntry *prev)
Iterate over a dictionary.
Definition: dict.c:42
CoreImageContext::list_filters
int list_filters
Option used to list all available filters including generators.
Definition: vf_coreimage.m:61