FFmpeg
dnn_interface.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * Implements DNN module initialization with specified backend.
24  */
25 
26 #include "../dnn_interface.h"
27 #include "libavutil/avassert.h"
28 #include "libavutil/mem.h"
29 #include "libavutil/opt.h"
30 #include "libavfilter/internal.h"
31 
33 extern const DNNModule ff_dnn_backend_tf;
34 extern const DNNModule ff_dnn_backend_torch;
35 
36 #define OFFSET(x) offsetof(DnnContext, x)
37 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
38 static const AVOption dnn_base_options[] = {
39  {"model", "path to model file",
40  OFFSET(model_filename), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, FLAGS},
41  {"input", "input name of the model",
42  OFFSET(model_inputname), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, FLAGS},
43  {"output", "output name of the model",
44  OFFSET(model_outputnames_string), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, FLAGS},
45  {"backend_configs", "backend configs (deprecated)",
46  OFFSET(backend_options), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, FLAGS | AV_OPT_FLAG_DEPRECATED},
47  {"options", "backend configs (deprecated)",
48  OFFSET(backend_options), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, FLAGS | AV_OPT_FLAG_DEPRECATED},
49  {"nireq", "number of request",
50  OFFSET(nireq), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, FLAGS},
51  {"async", "use DNN async inference",
52  OFFSET(async), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, FLAGS},
53  {"device", "device to run model",
54  OFFSET(device), AV_OPT_TYPE_STRING, {.str = NULL}, 0, 0, FLAGS},
55  {NULL}
56 };
57 
58 AVFILTER_DEFINE_CLASS(dnn_base);
59 
60 typedef struct DnnBackendInfo {
61  const size_t offset;
62  union {
63  const AVClass *class;
64  const DNNModule *module;
65  };
67 
69  {0, .class = &dnn_base_class},
70  // Must keep the same order as in DNNOptions, so offset value in incremental order
71 #if CONFIG_LIBTENSORFLOW
72  {offsetof(DnnContext, tf_option), .module = &ff_dnn_backend_tf},
73 #endif
74 #if CONFIG_LIBOPENVINO
75  {offsetof(DnnContext, ov_option), .module = &ff_dnn_backend_openvino},
76 #endif
77 #if CONFIG_LIBTORCH
78  {offsetof(DnnContext, torch_option), .module = &ff_dnn_backend_torch},
79 #endif
80 };
81 
82 const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
83 {
84  for (int i = 1; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
85  if (dnn_backend_info_list[i].module->type == backend_type)
87  }
88 
89  av_log(log_ctx, AV_LOG_ERROR,
90  "Module backend_type %d is not supported or enabled.\n",
91  backend_type);
92  return NULL;
93 }
94 
96 {
97  for (int i = 0; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
98  const AVClass **ptr = (const AVClass **) ((char *) ctx + dnn_backend_info_list[i].offset);
100  }
101 }
102 
103 void *ff_dnn_child_next(DnnContext *obj, void *prev) {
104  size_t pre_offset;
105 
106  if (!prev) {
107  av_assert0(obj->clazz);
108  return obj;
109  }
110 
111  pre_offset = (char *)prev - (char *)obj;
112  for (int i = 0; i < FF_ARRAY_ELEMS(dnn_backend_info_list) - 1; i++) {
113  if (dnn_backend_info_list[i].offset == pre_offset) {
114  const AVClass **ptr = (const AVClass **) ((char *) obj + dnn_backend_info_list[i + 1].offset);
115  av_assert0(*ptr);
116  return ptr;
117  }
118  }
119 
120  return NULL;
121 }
122 
123 const AVClass *ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t backend_mask)
124 {
125  for (uintptr_t i = (uintptr_t)*iter; i < FF_ARRAY_ELEMS(dnn_backend_info_list); i++) {
126  if (i > 0) {
127  const DNNModule *module = dnn_backend_info_list[i].module;
128 
129  if (!(module->type & backend_mask))
130  continue;
131  }
132 
133  *iter = (void *)(i + 1);
134  return dnn_backend_info_list[i].class;
135  }
136 
137  return NULL;
138 }
139 
ff_dnn_backend_openvino
const DNNModule ff_dnn_backend_openvino
DNNModule::type
DNNBackendType type
Definition: dnn_interface.h:177
opt.h
AVOption
AVOption.
Definition: opt.h:357
DnnContext::clazz
const AVClass * clazz
Definition: dnn_interface.h:144
dnn_base_options
static const AVOption dnn_base_options[]
Definition: dnn_interface.c:38
DnnContext
Definition: dnn_interface.h:143
dnn_backend_info_list
static const DnnBackendInfo dnn_backend_info_list[]
Definition: dnn_interface.c:68
avassert.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
FF_ARRAY_ELEMS
#define FF_ARRAY_ELEMS(a)
Definition: sinewin_tablegen.c:29
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:40
ff_dnn_init_child_class
void ff_dnn_init_child_class(DnnContext *ctx)
Definition: dnn_interface.c:95
ctx
AVFormatContext * ctx
Definition: movenc.c:49
DnnBackendInfo::offset
const size_t offset
Definition: dnn_interface.c:61
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(dnn_base)
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
DnnBackendInfo::module
const DNNModule * module
Definition: dnn_interface.c:64
NULL
#define NULL
Definition: coverity.c:32
FLAGS
#define FLAGS
Definition: dnn_interface.c:37
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DnnBackendInfo
Definition: dnn_interface.c:60
offset
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
Definition: writing_filters.txt:86
ff_get_dnn_module
const DNNModule * ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
Definition: dnn_interface.c:82
internal.h
ff_dnn_backend_tf
const DNNModule ff_dnn_backend_tf
Definition: dnn_backend_tf.c:887
ff_dnn_child_next
void * ff_dnn_child_next(DnnContext *obj, void *prev)
Definition: dnn_interface.c:103
AV_OPT_FLAG_DEPRECATED
#define AV_OPT_FLAG_DEPRECATED
Set if option is deprecated, users should refer to AVOption.help text for more information.
Definition: opt.h:314
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:256
ff_dnn_child_class_iterate_with_mask
const AVClass * ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t backend_mask)
Definition: dnn_interface.c:123
ff_dnn_backend_torch
const DNNModule ff_dnn_backend_torch
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Definition: opt.h:245
mem.h
DnnBackendInfo::class
const AVClass * class
Definition: dnn_interface.c:63
AV_OPT_TYPE_BOOL
@ AV_OPT_TYPE_BOOL
Definition: opt.h:261
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
OFFSET
#define OFFSET(x)
Definition: dnn_interface.c:36
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:249
DNNModule
Definition: dnn_interface.h:175