FFmpeg
vf_libvmaf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2017 Ronald S. Bultje <rsbultje@gmail.com>
3  * Copyright (c) 2017 Ashish Pratap Singh <ashk43712@gmail.com>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * Calculate the VMAF between two input videos.
25  */
26 
27 #include "config_components.h"
28 
29 #include <libvmaf.h>
30 
31 #include "libavutil/avstring.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/pixdesc.h"
34 #include "avfilter.h"
35 #include "drawutils.h"
36 #include "formats.h"
37 #include "framesync.h"
38 #include "internal.h"
39 #include "video.h"
40 
41 #if CONFIG_LIBVMAF_CUDA_FILTER
42 #include <libvmaf_cuda.h>
43 
44 #include "libavutil/hwcontext.h"
46 #endif
47 
48 typedef struct LIBVMAFContext {
49  const AVClass *class;
51  char *log_path;
52  char *log_fmt;
53  char *pool;
54  int n_threads;
56  char *model_cfg;
57  char *feature_cfg;
58  VmafContext *vmaf;
59  VmafModel **model;
60  unsigned model_cnt;
61  unsigned frame_cnt;
62  unsigned bpc;
63 #if CONFIG_LIBVMAF_CUDA_FILTER
64  VmafCudaState *cu_state;
65 #endif
67 
68 #define OFFSET(x) offsetof(LIBVMAFContext, x)
69 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
70 
71 static const AVOption libvmaf_options[] = {
72  {"log_path", "Set the file path to be used to write log.", OFFSET(log_path), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS},
73  {"log_fmt", "Set the format of the log (csv, json, xml, or sub).", OFFSET(log_fmt), AV_OPT_TYPE_STRING, {.str="xml"}, 0, 1, FLAGS},
74  {"pool", "Set the pool method to be used for computing vmaf.", OFFSET(pool), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS},
75  {"n_threads", "Set number of threads to be used when computing vmaf.", OFFSET(n_threads), AV_OPT_TYPE_INT, {.i64=0}, 0, UINT_MAX, FLAGS},
76  {"n_subsample", "Set interval for frame subsampling used when computing vmaf.", OFFSET(n_subsample), AV_OPT_TYPE_INT, {.i64=1}, 1, UINT_MAX, FLAGS},
77  {"model", "Set the model to be used for computing vmaf.", OFFSET(model_cfg), AV_OPT_TYPE_STRING, {.str="version=vmaf_v0.6.1"}, 0, 1, FLAGS},
78  {"feature", "Set the feature to be used for computing vmaf.", OFFSET(feature_cfg), AV_OPT_TYPE_STRING, {.str=NULL}, 0, 1, FLAGS},
79  { NULL }
80 };
81 
83 
84 static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt)
85 {
86  switch (av_pix_fmt) {
87  case AV_PIX_FMT_YUV420P:
91  return VMAF_PIX_FMT_YUV420P;
92  case AV_PIX_FMT_YUV422P:
96  return VMAF_PIX_FMT_YUV422P;
97  case AV_PIX_FMT_YUV444P:
101  return VMAF_PIX_FMT_YUV444P;
102  default:
103  return VMAF_PIX_FMT_UNKNOWN;
104  }
105 }
106 
107 static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc)
108 {
109  const int bytes_per_value = bpc > 8 ? 2 : 1;
110  int err = vmaf_picture_alloc(dst, pix_fmt_map(src->format), bpc,
111  src->width, src->height);
112  if (err)
113  return AVERROR(ENOMEM);
114 
115  for (unsigned i = 0; i < 3; i++) {
116  uint8_t *src_data = src->data[i];
117  uint8_t *dst_data = dst->data[i];
118  for (unsigned j = 0; j < dst->h[i]; j++) {
119  memcpy(dst_data, src_data, bytes_per_value * dst->w[i]);
120  src_data += src->linesize[i];
121  dst_data += dst->stride[i];
122  }
123  }
124 
125  return 0;
126 }
127 
128 static int do_vmaf(FFFrameSync *fs)
129 {
130  AVFilterContext *ctx = fs->parent;
131  LIBVMAFContext *s = ctx->priv;
132  VmafPicture pic_ref, pic_dist;
133  AVFrame *ref, *dist;
134  int err = 0;
135 
136  int ret = ff_framesync_dualinput_get(fs, &dist, &ref);
137  if (ret < 0)
138  return ret;
139  if (ctx->is_disabled || !ref)
140  return ff_filter_frame(ctx->outputs[0], dist);
141 
142  if (dist->color_range != ref->color_range) {
143  av_log(ctx, AV_LOG_WARNING, "distorted and reference "
144  "frames use different color ranges (%s != %s)\n",
146  av_color_range_name(ref->color_range));
147  }
148 
149  err = copy_picture_data(ref, &pic_ref, s->bpc);
150  if (err) {
151  av_log(s, AV_LOG_ERROR, "problem during vmaf_picture_alloc.\n");
152  return AVERROR(ENOMEM);
153  }
154 
155  err = copy_picture_data(dist, &pic_dist, s->bpc);
156  if (err) {
157  av_log(s, AV_LOG_ERROR, "problem during vmaf_picture_alloc.\n");
158  vmaf_picture_unref(&pic_ref);
159  return AVERROR(ENOMEM);
160  }
161 
162  err = vmaf_read_pictures(s->vmaf, &pic_ref, &pic_dist, s->frame_cnt++);
163  if (err) {
164  av_log(s, AV_LOG_ERROR, "problem during vmaf_read_pictures.\n");
165  return AVERROR(EINVAL);
166  }
167 
168  return ff_filter_frame(ctx->outputs[0], dist);
169 }
170 
171 static AVDictionary **delimited_dict_parse(char *str, unsigned *cnt)
172 {
173  AVDictionary **dict = NULL;
174  char *str_copy = NULL;
175  char *saveptr = NULL;
176  unsigned cnt2;
177  int err = 0;
178 
179  if (!str)
180  return NULL;
181 
182  cnt2 = 1;
183  for (char *p = str; *p; p++) {
184  if (*p == '|')
185  cnt2++;
186  }
187 
188  dict = av_calloc(cnt2, sizeof(*dict));
189  if (!dict)
190  goto fail;
191 
192  str_copy = av_strdup(str);
193  if (!str_copy)
194  goto fail;
195 
196  *cnt = 0;
197  for (unsigned i = 0; i < cnt2; i++) {
198  char *s = av_strtok(i == 0 ? str_copy : NULL, "|", &saveptr);
199  if (!s)
200  continue;
201  err = av_dict_parse_string(&dict[(*cnt)++], s, "=", ":", 0);
202  if (err)
203  goto fail;
204  }
205 
206  av_free(str_copy);
207  return dict;
208 
209 fail:
210  if (dict) {
211  for (unsigned i = 0; i < *cnt; i++) {
212  if (dict[i])
213  av_dict_free(&dict[i]);
214  }
215  av_free(dict);
216  }
217 
218  av_free(str_copy);
219  *cnt = 0;
220  return NULL;
221 }
222 
224 {
225  LIBVMAFContext *s = ctx->priv;
226  AVDictionary **dict = NULL;
227  unsigned dict_cnt;
228  int err = 0;
229 
230  if (!s->feature_cfg)
231  return 0;
232 
233  dict = delimited_dict_parse(s->feature_cfg, &dict_cnt);
234  if (!dict) {
236  "could not parse feature config: %s\n", s->feature_cfg);
237  return AVERROR(EINVAL);
238  }
239 
240  for (unsigned i = 0; i < dict_cnt; i++) {
241  char *feature_name = NULL;
242  VmafFeatureDictionary *feature_opts_dict = NULL;
243  const AVDictionaryEntry *e = NULL;
244 
245  while (e = av_dict_iterate(dict[i], e)) {
246  if (!strcmp(e->key, "name")) {
247  feature_name = e->value;
248  continue;
249  }
250 
251  err = vmaf_feature_dictionary_set(&feature_opts_dict, e->key,
252  e->value);
253  if (err) {
255  "could not set feature option: %s.%s=%s\n",
256  feature_name, e->key, e->value);
257  goto exit;
258  }
259  }
260 
261  err = vmaf_use_feature(s->vmaf, feature_name, feature_opts_dict);
262  if (err) {
264  "problem during vmaf_use_feature: %s\n", feature_name);
265  goto exit;
266  }
267  }
268 
269 exit:
270  for (unsigned i = 0; i < dict_cnt; i++) {
271  if (dict[i])
272  av_dict_free(&dict[i]);
273  }
274  av_free(dict);
275  return err;
276 }
277 
279 {
280  LIBVMAFContext *s = ctx->priv;
281  AVDictionary **dict;
282  unsigned dict_cnt;
283  int err = 0;
284 
285  if (!s->model_cfg) return 0;
286 
287  dict_cnt = 0;
288  dict = delimited_dict_parse(s->model_cfg, &dict_cnt);
289  if (!dict) {
291  "could not parse model config: %s\n", s->model_cfg);
292  return AVERROR(EINVAL);
293  }
294 
295  s->model_cnt = dict_cnt;
296  s->model = av_calloc(s->model_cnt, sizeof(*s->model));
297  if (!s->model)
298  return AVERROR(ENOMEM);
299 
300  for (unsigned i = 0; i < dict_cnt; i++) {
301  VmafModelConfig model_cfg = { 0 };
302  const AVDictionaryEntry *e = NULL;
303  char *version = NULL;
304  char *path = NULL;
305 
306  while (e = av_dict_iterate(dict[i], e)) {
307  if (!strcmp(e->key, "disable_clip")) {
308  model_cfg.flags |= !strcmp(e->value, "true") ?
309  VMAF_MODEL_FLAG_DISABLE_CLIP : 0;
310  continue;
311  }
312 
313  if (!strcmp(e->key, "enable_transform")) {
314  model_cfg.flags |= !strcmp(e->value, "true") ?
315  VMAF_MODEL_FLAG_ENABLE_TRANSFORM : 0;
316  continue;
317  }
318 
319  if (!strcmp(e->key, "name")) {
320  model_cfg.name = e->value;
321  continue;
322  }
323 
324  if (!strcmp(e->key, "version")) {
325  version = e->value;
326  continue;
327  }
328 
329  if (!strcmp(e->key, "path")) {
330  path = e->value;
331  continue;
332  }
333  }
334 
335  if (version) {
336  err = vmaf_model_load(&s->model[i], &model_cfg, version);
337  if (err) {
339  "could not load libvmaf model with version: %s\n",
340  version);
341  goto exit;
342  }
343  }
344 
345  if (path && !s->model[i]) {
346  err = vmaf_model_load_from_path(&s->model[i], &model_cfg, path);
347  if (err) {
349  "could not load libvmaf model with path: %s\n",
350  path);
351  goto exit;
352  }
353  }
354 
355  if (!s->model[i]) {
357  "could not load libvmaf model with config: %s\n",
358  s->model_cfg);
359  goto exit;
360  }
361 
362  while (e = av_dict_iterate(dict[i], e)) {
363  VmafFeatureDictionary *feature_opts_dict = NULL;
364  char *feature_opt = NULL;
365 
366  char *feature_name = av_strtok(e->key, ".", &feature_opt);
367  if (!feature_opt)
368  continue;
369 
370  err = vmaf_feature_dictionary_set(&feature_opts_dict,
371  feature_opt, e->value);
372  if (err) {
374  "could not set feature option: %s.%s=%s\n",
375  feature_name, feature_opt, e->value);
376  err = AVERROR(EINVAL);
377  goto exit;
378  }
379 
380  err = vmaf_model_feature_overload(s->model[i], feature_name,
381  feature_opts_dict);
382  if (err) {
384  "could not overload feature: %s\n", feature_name);
385  err = AVERROR(EINVAL);
386  goto exit;
387  }
388  }
389  }
390 
391  for (unsigned i = 0; i < s->model_cnt; i++) {
392  err = vmaf_use_features_from_model(s->vmaf, s->model[i]);
393  if (err) {
395  "problem during vmaf_use_features_from_model\n");
396  err = AVERROR(EINVAL);
397  goto exit;
398  }
399  }
400 
401 exit:
402  for (unsigned i = 0; i < dict_cnt; i++) {
403  if (dict[i])
404  av_dict_free(&dict[i]);
405  }
406  av_free(dict);
407  return err;
408 }
409 
410 static enum VmafLogLevel log_level_map(int log_level)
411 {
412  switch (log_level) {
413  case AV_LOG_QUIET:
414  return VMAF_LOG_LEVEL_NONE;
415  case AV_LOG_ERROR:
416  return VMAF_LOG_LEVEL_ERROR;
417  case AV_LOG_WARNING:
418  return VMAF_LOG_LEVEL_WARNING;
419  case AV_LOG_INFO:
420  return VMAF_LOG_LEVEL_INFO;
421  case AV_LOG_DEBUG:
422  return VMAF_LOG_LEVEL_DEBUG;
423  default:
424  return VMAF_LOG_LEVEL_INFO;
425  }
426 }
427 
429 {
430  LIBVMAFContext *s = ctx->priv;
431  int err = 0;
432 
433  VmafConfiguration cfg = {
434  .log_level = log_level_map(av_log_get_level()),
435  .n_subsample = s->n_subsample,
436  .n_threads = s->n_threads,
437  };
438 
439  err = vmaf_init(&s->vmaf, cfg);
440  if (err)
441  return AVERROR(EINVAL);
442 
443  err = parse_models(ctx);
444  if (err)
445  return err;
446 
447  err = parse_features(ctx);
448  if (err)
449  return err;
450 
451  s->fs.on_event = do_vmaf;
452  return 0;
453 }
454 
455 static const enum AVPixelFormat pix_fmts[] = {
461 };
462 
464 {
465  AVFilterContext *ctx = inlink->dst;
466  LIBVMAFContext *s = ctx->priv;
467  const AVPixFmtDescriptor *desc;
468  int err = 0;
469 
470  if (ctx->inputs[0]->w != ctx->inputs[1]->w) {
471  av_log(ctx, AV_LOG_ERROR, "input width must match.\n");
472  err |= AVERROR(EINVAL);
473  }
474 
475  if (ctx->inputs[0]->h != ctx->inputs[1]->h) {
476  av_log(ctx, AV_LOG_ERROR, "input height must match.\n");
477  err |= AVERROR(EINVAL);
478  }
479 
480  if (ctx->inputs[0]->format != ctx->inputs[1]->format) {
481  av_log(ctx, AV_LOG_ERROR, "input pix_fmt must match.\n");
482  err |= AVERROR(EINVAL);
483  }
484 
485  if (err)
486  return err;
487 
488  desc = av_pix_fmt_desc_get(inlink->format);
489  s->bpc = desc->comp[0].depth;
490 
491  return 0;
492 }
493 
494 static int config_output(AVFilterLink *outlink)
495 {
496  AVFilterContext *ctx = outlink->src;
497  LIBVMAFContext *s = ctx->priv;
498  AVFilterLink *mainlink = ctx->inputs[0];
499  int ret;
500 
502  if (ret < 0)
503  return ret;
504  outlink->w = mainlink->w;
505  outlink->h = mainlink->h;
506  outlink->time_base = mainlink->time_base;
507  outlink->sample_aspect_ratio = mainlink->sample_aspect_ratio;
508  outlink->frame_rate = mainlink->frame_rate;
509  if ((ret = ff_framesync_configure(&s->fs)) < 0)
510  return ret;
511 
512  return 0;
513 }
514 
516 {
517  LIBVMAFContext *s = ctx->priv;
518  return ff_framesync_activate(&s->fs);
519 }
520 
521 static enum VmafOutputFormat log_fmt_map(const char *log_fmt)
522 {
523  if (log_fmt) {
524  if (!strcmp(log_fmt, "xml"))
525  return VMAF_OUTPUT_FORMAT_XML;
526  if (!strcmp(log_fmt, "json"))
527  return VMAF_OUTPUT_FORMAT_JSON;
528  if (!strcmp(log_fmt, "csv"))
529  return VMAF_OUTPUT_FORMAT_CSV;
530  if (!strcmp(log_fmt, "sub"))
531  return VMAF_OUTPUT_FORMAT_SUB;
532  }
533 
534  return VMAF_OUTPUT_FORMAT_XML;
535 }
536 
537 static enum VmafPoolingMethod pool_method_map(const char *pool_method)
538 {
539  if (pool_method) {
540  if (!strcmp(pool_method, "min"))
541  return VMAF_POOL_METHOD_MIN;
542  if (!strcmp(pool_method, "mean"))
543  return VMAF_POOL_METHOD_MEAN;
544  if (!strcmp(pool_method, "harmonic_mean"))
545  return VMAF_POOL_METHOD_HARMONIC_MEAN;
546  }
547 
548  return VMAF_POOL_METHOD_MEAN;
549 }
550 
552 {
553  LIBVMAFContext *s = ctx->priv;
554  int err = 0;
555 
556  ff_framesync_uninit(&s->fs);
557 
558  if (!s->frame_cnt)
559  goto clean_up;
560 
561  err = vmaf_read_pictures(s->vmaf, NULL, NULL, 0);
562  if (err) {
564  "problem flushing libvmaf context.\n");
565  }
566 
567  for (unsigned i = 0; i < s->model_cnt; i++) {
568  double vmaf_score;
569  err = vmaf_score_pooled(s->vmaf, s->model[i], pool_method_map(s->pool),
570  &vmaf_score, 0, s->frame_cnt - 1);
571  if (err) {
573  "problem getting pooled vmaf score.\n");
574  }
575 
576  av_log(ctx, AV_LOG_INFO, "VMAF score: %f\n", vmaf_score);
577  }
578 
579  if (s->vmaf) {
580  if (s->log_path && !err)
581  vmaf_write_output(s->vmaf, s->log_path, log_fmt_map(s->log_fmt));
582  }
583 
584 clean_up:
585  if (s->model) {
586  for (unsigned i = 0; i < s->model_cnt; i++) {
587  if (s->model[i])
588  vmaf_model_destroy(s->model[i]);
589  }
590  av_free(s->model);
591  }
592 
593  if (s->vmaf)
594  vmaf_close(s->vmaf);
595 }
596 
597 static const AVFilterPad libvmaf_inputs[] = {
598  {
599  .name = "main",
600  .type = AVMEDIA_TYPE_VIDEO,
601  },
602  {
603  .name = "reference",
604  .type = AVMEDIA_TYPE_VIDEO,
605  .config_props = config_input_ref,
606  },
607 };
608 
609 static const AVFilterPad libvmaf_outputs[] = {
610  {
611  .name = "default",
612  .type = AVMEDIA_TYPE_VIDEO,
613  .config_props = config_output,
614  },
615 };
616 
618  .name = "libvmaf",
619  .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF between two video streams."),
620  .preinit = libvmaf_framesync_preinit,
621  .init = init,
622  .uninit = uninit,
623  .activate = activate,
624  .priv_size = sizeof(LIBVMAFContext),
625  .priv_class = &libvmaf_class,
629 };
630 
631 #if CONFIG_LIBVMAF_CUDA_FILTER
632 static const enum AVPixelFormat supported_formats[] = {
635 };
636 
637 static int format_is_supported(enum AVPixelFormat fmt)
638 {
639  int i;
640 
641  for (i = 0; i < FF_ARRAY_ELEMS(supported_formats); i++)
642  if (supported_formats[i] == fmt)
643  return 1;
644  return 0;
645 }
646 
647 static int config_props_cuda(AVFilterLink *outlink)
648 {
649  int err;
650  AVFilterContext *ctx = outlink->src;
651  LIBVMAFContext *s = ctx->priv;
652  AVFilterLink *inlink = ctx->inputs[0];
653  AVHWFramesContext *frames_ctx = (AVHWFramesContext*) inlink->hw_frames_ctx->data;
654  AVCUDADeviceContext *device_hwctx = frames_ctx->device_ctx->hwctx;
655  CUcontext cu_ctx = device_hwctx->cuda_ctx;
657 
658  VmafConfiguration cfg = {
659  .log_level = log_level_map(av_log_get_level()),
660  .n_subsample = s->n_subsample,
661  .n_threads = s->n_threads,
662  };
663 
664  VmafCudaPictureConfiguration cuda_pic_cfg = {
665  .pic_params = {
666  .bpc = desc->comp[0].depth,
667  .w = inlink->w,
668  .h = inlink->h,
669  .pix_fmt = pix_fmt_map(frames_ctx->sw_format),
670  },
671  .pic_prealloc_method = VMAF_CUDA_PICTURE_PREALLOCATION_METHOD_DEVICE,
672  };
673 
674  VmafCudaConfiguration cuda_cfg = {
675  .cu_ctx = cu_ctx,
676  };
677 
678  if (!format_is_supported(frames_ctx->sw_format)) {
680  "Unsupported input format: %s\n", desc->name);
681  return AVERROR(EINVAL);
682  }
683 
684  err = vmaf_init(&s->vmaf, cfg);
685  if (err)
686  return AVERROR(EINVAL);
687 
688  err = vmaf_cuda_state_init(&s->cu_state, cuda_cfg);
689  if (err)
690  return AVERROR(EINVAL);
691 
692  err = vmaf_cuda_import_state(s->vmaf, s->cu_state);
693  if (err)
694  return AVERROR(EINVAL);
695 
696  err = vmaf_cuda_preallocate_pictures(s->vmaf, cuda_pic_cfg);
697  if (err < 0)
698  return err;
699 
700  err = parse_models(ctx);
701  if (err)
702  return err;
703 
704  err = parse_features(ctx);
705  if (err)
706  return err;
707 
708  return config_output(outlink);
709 }
710 
711 static int copy_picture_data_cuda(VmafContext* vmaf,
712  AVCUDADeviceContext* device_hwctx,
713  AVFrame* src, VmafPicture* dst,
714  enum AVPixelFormat pix_fmt)
715 {
717  CudaFunctions *cu = device_hwctx->internal->cuda_dl;
718 
719  CUDA_MEMCPY2D m = {
720  .srcMemoryType = CU_MEMORYTYPE_DEVICE,
721  .dstMemoryType = CU_MEMORYTYPE_DEVICE,
722  };
723 
724  int err = vmaf_cuda_fetch_preallocated_picture(vmaf, dst);
725  if (err)
726  return AVERROR(ENOMEM);
727 
728  err = cu->cuCtxPushCurrent(device_hwctx->cuda_ctx);
729  if (err)
730  return AVERROR_EXTERNAL;
731 
732  for (unsigned i = 0; i < pix_desc->nb_components; i++) {
733  m.srcDevice = (CUdeviceptr) src->data[i];
734  m.srcPitch = src->linesize[i];
735  m.dstDevice = (CUdeviceptr) dst->data[i];
736  m.dstPitch = dst->stride[i];
737  m.WidthInBytes = dst->w[i] * ((dst->bpc + 7) / 8);
738  m.Height = dst->h[i];
739 
740  err = cu->cuMemcpy2D(&m);
741  if (err)
742  return AVERROR_EXTERNAL;
743  break;
744  }
745 
746  err = cu->cuCtxPopCurrent(NULL);
747  if (err)
748  return AVERROR_EXTERNAL;
749 
750  return 0;
751 }
752 
753 static int do_vmaf_cuda(FFFrameSync* fs)
754 {
755  AVFilterContext* ctx = fs->parent;
756  LIBVMAFContext* s = ctx->priv;
757  AVFilterLink *inlink = ctx->inputs[0];
758  AVHWFramesContext *frames_ctx = (AVHWFramesContext*) inlink->hw_frames_ctx->data;
759  AVCUDADeviceContext *device_hwctx = frames_ctx->device_ctx->hwctx;
760  VmafPicture pic_ref, pic_dist;
761  AVFrame *ref, *dist;
762 
763  int err = 0;
764 
765  err = ff_framesync_dualinput_get(fs, &dist, &ref);
766  if (err < 0)
767  return err;
768  if (ctx->is_disabled || !ref)
769  return ff_filter_frame(ctx->outputs[0], dist);
770 
771  err = copy_picture_data_cuda(s->vmaf, device_hwctx, ref, &pic_ref,
772  frames_ctx->sw_format);
773  if (err) {
774  av_log(s, AV_LOG_ERROR, "problem during copy_picture_data_cuda.\n");
775  return AVERROR(ENOMEM);
776  }
777 
778  err = copy_picture_data_cuda(s->vmaf, device_hwctx, dist, &pic_dist,
779  frames_ctx->sw_format);
780  if (err) {
781  av_log(s, AV_LOG_ERROR, "problem during copy_picture_data_cuda.\n");
782  return AVERROR(ENOMEM);
783  }
784 
785  err = vmaf_read_pictures(s->vmaf, &pic_ref, &pic_dist, s->frame_cnt++);
786  if (err) {
787  av_log(s, AV_LOG_ERROR, "problem during vmaf_read_pictures.\n");
788  return AVERROR(EINVAL);
789  }
790 
791  return ff_filter_frame(ctx->outputs[0], dist);
792 }
793 
794 static av_cold int init_cuda(AVFilterContext *ctx)
795 {
796  LIBVMAFContext *s = ctx->priv;
797  s->fs.on_event = do_vmaf_cuda;
798  return 0;
799 }
800 
801 static const AVFilterPad libvmaf_outputs_cuda[] = {
802  {
803  .name = "default",
804  .type = AVMEDIA_TYPE_VIDEO,
805  .config_props = config_props_cuda,
806  },
807 };
808 
810  .name = "libvmaf_cuda",
811  .description = NULL_IF_CONFIG_SMALL("Calculate the VMAF between two video streams."),
812  .preinit = libvmaf_framesync_preinit,
813  .init = init_cuda,
814  .uninit = uninit,
815  .activate = activate,
816  .priv_size = sizeof(LIBVMAFContext),
817  .priv_class = &libvmaf_class,
819  FILTER_OUTPUTS(libvmaf_outputs_cuda),
821  .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE,
822 };
823 #endif
AVHWDeviceContext::hwctx
void * hwctx
The format-specific data, allocated and freed by libavutil along with this context.
Definition: hwcontext.h:85
ff_framesync_configure
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:134
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
AVFrame::color_range
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: frame.h:619
AV_PIX_FMT_CUDA
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition: pixfmt.h:260
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:71
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
hwcontext_cuda_internal.h
libvmaf_inputs
static const AVFilterPad libvmaf_inputs[]
Definition: vf_libvmaf.c:597
ff_framesync_uninit
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:304
FF_FILTER_FLAG_HWFRAME_AWARE
#define FF_FILTER_FLAG_HWFRAME_AWARE
The filter is aware of hardware frames, and any hardware frame context should not be automatically pr...
Definition: internal.h:351
AV_LOG_QUIET
#define AV_LOG_QUIET
Print no output.
Definition: log.h:162
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1018
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2962
FILTER_PIXFMTS_ARRAY
#define FILTER_PIXFMTS_ARRAY(array)
Definition: internal.h:162
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
LIBVMAFContext
Definition: vf_libvmaf.c:48
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:340
pixdesc.h
log_fmt_map
static enum VmafOutputFormat log_fmt_map(const char *log_fmt)
Definition: vf_libvmaf.c:521
AVOption
AVOption.
Definition: opt.h:346
AV_PIX_FMT_YUV420P16LE
@ AV_PIX_FMT_YUV420P16LE
planar YUV 4:2:0, 24bpp, (1 Cr & Cb sample per 2x2 Y samples), little-endian
Definition: pixfmt.h:128
AVDictionary
Definition: dict.c:34
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:170
FFFrameSync
Frame sync structure.
Definition: framesync.h:168
video.h
AV_PIX_FMT_YUV444P16LE
@ AV_PIX_FMT_YUV444P16LE
planar YUV 4:4:4, 48bpp, (1 Cr & Cb sample per 1x1 Y samples), little-endian
Definition: pixfmt.h:132
formats.h
AV_PIX_FMT_YUV420P12LE
@ AV_PIX_FMT_YUV420P12LE
planar YUV 4:2:0,18bpp, (1 Cr & Cb sample per 2x2 Y samples), little-endian
Definition: pixfmt.h:268
fail
#define fail()
Definition: checkasm.h:179
LIBVMAFContext::fs
FFFrameSync fs
Definition: vf_libvmaf.c:50
LIBVMAFContext::n_subsample
int n_subsample
Definition: vf_libvmaf.c:55
AVCUDADeviceContext::cuda_ctx
CUcontext cuda_ctx
Definition: hwcontext_cuda.h:43
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:33
AV_PIX_FMT_YUV420P10LE
@ AV_PIX_FMT_YUV420P10LE
planar YUV 4:2:0, 15bpp, (1 Cr & Cb sample per 2x2 Y samples), little-endian
Definition: pixfmt.h:156
AV_PIX_FMT_YUV444P12LE
@ AV_PIX_FMT_YUV444P12LE
planar YUV 4:4:4,36bpp, (1 Cr & Cb sample per 1x1 Y samples), little-endian
Definition: pixfmt.h:276
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
av_cold
#define av_cold
Definition: attributes.h:90
s
#define s(width, name)
Definition: cbs_vp9.c:198
AV_PIX_FMT_YUV444P16
#define AV_PIX_FMT_YUV444P16
Definition: pixfmt.h:491
LIBVMAFContext::model
VmafModel ** model
Definition: vf_libvmaf.c:59
pix_fmt
static enum AVPixelFormat pix_fmt
Definition: demux_decode.c:41
ff_vf_libvmaf
const AVFilter ff_vf_libvmaf
Definition: vf_libvmaf.c:617
config_input_ref
static int config_input_ref(AVFilterLink *inlink)
Definition: vf_libvmaf.c:463
AVDictionaryEntry::key
char * key
Definition: dict.h:90
av_strtok
char * av_strtok(char *s, const char *delim, char **saveptr)
Split the string into several tokens which can be accessed by successive calls to av_strtok().
Definition: avstring.c:178
copy_picture_data
static int copy_picture_data(AVFrame *src, VmafPicture *dst, unsigned bpc)
Definition: vf_libvmaf.c:107
AV_LOG_DEBUG
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:201
ctx
AVFormatContext * ctx
Definition: movenc.c:48
format_is_supported
static int format_is_supported(enum AVPixelFormat fmt)
Definition: vf_bilateral_cuda.c:143
LIBVMAFContext::model_cnt
unsigned model_cnt
Definition: vf_libvmaf.c:60
LIBVMAFContext::frame_cnt
unsigned frame_cnt
Definition: vf_libvmaf.c:61
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:73
FRAMESYNC_DEFINE_CLASS
FRAMESYNC_DEFINE_CLASS(libvmaf, LIBVMAFContext, fs)
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:182
AV_PIX_FMT_YUV444P10LE
@ AV_PIX_FMT_YUV444P10LE
planar YUV 4:4:4, 30bpp, (1 Cr & Cb sample per 1x1 Y samples), little-endian
Definition: pixfmt.h:162
LIBVMAFContext::vmaf
VmafContext * vmaf
Definition: vf_libvmaf.c:58
ff_vf_libvmaf_cuda
const AVFilter ff_vf_libvmaf_cuda
activate
static int activate(AVFilterContext *ctx)
Definition: vf_libvmaf.c:515
av_color_range_name
const char * av_color_range_name(enum AVColorRange range)
Definition: pixdesc.c:3278
av_log_get_level
int av_log_get_level(void)
Get the current log level.
Definition: log.c:442
AV_PIX_FMT_YUV422P16LE
@ AV_PIX_FMT_YUV422P16LE
planar YUV 4:2:2, 32bpp, (1 Cr & Cb sample per 2x1 Y samples), little-endian
Definition: pixfmt.h:130
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
AVHWFramesContext::sw_format
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition: hwcontext.h:210
do_vmaf
static int do_vmaf(FFFrameSync *fs)
Definition: vf_libvmaf.c:128
fs
#define fs(width, name, subs,...)
Definition: cbs_vp9.c:200
AVPixFmtDescriptor::nb_components
uint8_t nb_components
The number of components each pixel has, (1-4)
Definition: pixdesc.h:71
LIBVMAFContext::log_fmt
char * log_fmt
Definition: vf_libvmaf.c:52
uninit
static av_cold void uninit(AVFilterContext *ctx)
Definition: vf_libvmaf.c:551
LIBVMAFContext::feature_cfg
char * feature_cfg
Definition: vf_libvmaf.c:57
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: vf_libvmaf.c:455
pool_method_map
static enum VmafPoolingMethod pool_method_map(const char *pool_method)
Definition: vf_libvmaf.c:537
config_output
static int config_output(AVFilterLink *outlink)
Definition: vf_libvmaf.c:494
AVCUDADeviceContext::internal
AVCUDADeviceContextInternal * internal
Definition: hwcontext_cuda.h:45
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:106
ff_framesync_init_dualinput
int ff_framesync_init_dualinput(FFFrameSync *fs, AVFilterContext *parent)
Initialize a frame sync structure for dualinput.
Definition: framesync.c:375
AV_PIX_FMT_YUV422P10LE
@ AV_PIX_FMT_YUV422P10LE
planar YUV 4:2:2, 20bpp, (1 Cr & Cb sample per 2x1 Y samples), little-endian
Definition: pixfmt.h:158
LIBVMAFContext::n_threads
int n_threads
Definition: vf_libvmaf.c:54
LIBVMAFContext::log_path
char * log_path
Definition: vf_libvmaf.c:51
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:59
av_dict_free
void av_dict_free(AVDictionary **pm)
Free all the memory allocated for an AVDictionary struct and all keys and values.
Definition: dict.c:223
supported_formats
static enum AVPixelFormat supported_formats[]
Definition: vf_bilateral_cuda.c:37
LIBVMAFContext::model_cfg
char * model_cfg
Definition: vf_libvmaf.c:56
version
version
Definition: libkvazaar.c:321
LIBVMAFContext::bpc
unsigned bpc
Definition: vf_libvmaf.c:62
AV_LOG_INFO
#define AV_LOG_INFO
Standard information.
Definition: log.h:191
delimited_dict_parse
static AVDictionary ** delimited_dict_parse(char *str, unsigned *cnt)
Definition: vf_libvmaf.c:171
internal.h
FILTER_SINGLE_PIXFMT
#define FILTER_SINGLE_PIXFMT(pix_fmt_)
Definition: internal.h:172
AVCUDADeviceContextInternal::cuda_dl
CudaFunctions * cuda_dl
Definition: hwcontext_cuda_internal.h:32
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:255
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:39
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:262
AVFilter
Filter definition.
Definition: avfilter.h:166
AVHWFramesContext
This struct describes a set or pool of "hardware" frames (i.e.
Definition: hwcontext.h:115
AVCUDADeviceContext
This struct is allocated as AVHWDeviceContext.hwctx.
Definition: hwcontext_cuda.h:42
ret
ret
Definition: filter_design.txt:187
AVHWFramesContext::device_ctx
AVHWDeviceContext * device_ctx
The parent AVHWDeviceContext.
Definition: hwcontext.h:134
libvmaf_outputs
static const AVFilterPad libvmaf_outputs[]
Definition: vf_libvmaf.c:609
pix_fmt_map
static enum VmafPixelFormat pix_fmt_map(enum AVPixelFormat av_pix_fmt)
Definition: vf_libvmaf.c:84
framesync.h
libvmaf_options
static const AVOption libvmaf_options[]
Definition: vf_libvmaf.c:71
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:72
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Definition: opt.h:235
avfilter.h
av_dict_parse_string
int av_dict_parse_string(AVDictionary **pm, const char *str, const char *key_val_sep, const char *pairs_sep, int flags)
Parse the key/value pairs list and add the parsed entries to a dictionary.
Definition: dict.c:200
ref
static int ref[MAX_W *MAX_W]
Definition: jpeg2000dwt.c:112
LIBVMAFContext::pool
char * pool
Definition: vf_libvmaf.c:53
OFFSET
#define OFFSET(x)
Definition: vf_libvmaf.c:68
FLAGS
#define FLAGS
Definition: vf_libvmaf.c:69
AV_PIX_FMT_YUV444P
@ AV_PIX_FMT_YUV444P
planar YUV 4:4:4, 24bpp, (1 Cr & Cb sample per 1x1 Y samples)
Definition: pixfmt.h:78
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
av_strdup
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:270
desc
const char * desc
Definition: libsvtav1.c:73
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
AV_PIX_FMT_YUV422P
@ AV_PIX_FMT_YUV422P
planar YUV 4:2:2, 16bpp, (1 Cr & Cb sample per 2x1 Y samples)
Definition: pixfmt.h:77
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
init
static av_cold int init(AVFilterContext *ctx)
Definition: vf_libvmaf.c:428
av_free
#define av_free(p)
Definition: tableprint_vlc.h:33
AVDictionaryEntry
Definition: dict.h:89
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:183
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
log_level_map
static enum VmafLogLevel log_level_map(int log_level)
Definition: vf_libvmaf.c:410
hwcontext.h
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
parse_models
static int parse_models(AVFilterContext *ctx)
Definition: vf_libvmaf.c:278
AVDictionaryEntry::value
char * value
Definition: dict.h:91
ff_framesync_activate
int ff_framesync_activate(FFFrameSync *fs)
Examine the frames in the filter's input and try to produce output.
Definition: framesync.c:355
avstring.h
ff_framesync_dualinput_get
int ff_framesync_dualinput_get(FFFrameSync *fs, AVFrame **f0, AVFrame **f1)
Definition: framesync.c:393
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:239
drawutils.h
parse_features
static int parse_features(AVFilterContext *ctx)
Definition: vf_libvmaf.c:223
AV_PIX_FMT_YUV422P12LE
@ AV_PIX_FMT_YUV422P12LE
planar YUV 4:2:2,24bpp, (1 Cr & Cb sample per 2x1 Y samples), little-endian
Definition: pixfmt.h:272
av_dict_iterate
const AVDictionaryEntry * av_dict_iterate(const AVDictionary *m, const AVDictionaryEntry *prev)
Iterate over a dictionary.
Definition: dict.c:44