FFmpeg
dnn_interface.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN inference engine interface.
24  */
25 
26 #ifndef AVFILTER_DNN_INTERFACE_H
27 #define AVFILTER_DNN_INTERFACE_H
28 
29 #include <stdint.h>
30 #include "libavutil/frame.h"
31 #include "avfilter.h"
32 
33 #define DNN_GENERIC_ERROR FFERRTAG('D','N','N','!')
34 
35 typedef enum {
36  DNN_TF = 1,
37  DNN_OV = 1 << 1,
38  DNN_TH = 1 << 2
40 
41 typedef enum {DNN_FLOAT = 1, DNN_UINT8 = 4} DNNDataType;
42 
43 typedef enum {
48 
49 typedef enum {
50  DAST_FAIL, // something wrong
51  DAST_EMPTY_QUEUE, // no more inference result to get
52  DAST_NOT_READY, // all queued inferences are not finished
53  DAST_SUCCESS // got a result frame successfully
55 
56 typedef enum {
58  DFT_PROCESS_FRAME, // process the whole frame
59  DFT_ANALYTICS_DETECT, // detect from the whole frame
60  DFT_ANALYTICS_CLASSIFY, // classify for each bounding box
62 
63 typedef enum {
67 } DNNLayout;
68 
69 typedef struct DNNData{
70  void *data;
71  int dims[4];
72  // dt and order together decide the color format
76  float scale;
77  float mean;
78 } DNNData;
79 
80 typedef struct DNNExecBaseParams {
81  const char *input_name;
82  const char **output_names;
83  uint32_t nb_output;
87 
90  const char *target;
92 
95 typedef int (*ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx);
96 
97 typedef struct DNNModel{
98  // Stores FilterContext used for the interaction between AVFrame and DNNData
100  // Stores function type of the model
102  // Gets model input information
103  // Just reuse struct DNNData here, actually the DNNData.data field is not needed.
104  int (*get_input)(struct DNNModel *model, DNNData *input, const char *input_name);
105  // Gets model output width/height with given input w/h
106  int (*get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height,
107  const char *output_name, int *output_width, int *output_height);
108  // set the pre process to transfer data from AVFrame to DNNData
109  // the default implementation within DNN is used if it is not provided by the filter
111  // set the post process to transfer data from DNNData to AVFrame
112  // the default implementation within DNN is used if it is not provided by the filter
114  // set the post process to interpret detect result from DNNData
116  // set the post process to interpret classify result from DNNData
118 } DNNModel;
119 
120 typedef struct TFOptions{
121  const AVClass *clazz;
122 
123  char *sess_config;
124 } TFOptions;
125 
126 typedef struct OVOptions {
127  const AVClass *clazz;
128 
132  float scale;
133  float mean;
134 } OVOptions;
135 
136 typedef struct THOptions {
137  const AVClass *clazz;
138  int optimize;
139 } THOptions;
140 
141 typedef struct DNNModule DNNModule;
142 
143 typedef struct DnnContext {
144  const AVClass *clazz;
145 
147 
153  int async;
154 
156  uint32_t nb_outputs;
158 
159  int nireq;
160  char *device;
161 
162 #if CONFIG_LIBTENSORFLOW
163  TFOptions tf_option;
164 #endif
165 
166 #if CONFIG_LIBOPENVINO
167  OVOptions ov_option;
168 #endif
169 #if CONFIG_LIBTORCH
170  THOptions torch_option;
171 #endif
172 } DnnContext;
173 
174 // Stores pointers to functions for loading, executing, freeing DNN models for one of the backends.
175 struct DNNModule {
176  const AVClass clazz;
178  // Loads model and parameters from given file. Returns NULL if it is not possible.
180  // Executes model with specified input and output. Returns the error code otherwise.
181  int (*execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params);
182  // Retrieve inference result.
184  // Flush all the pending tasks.
185  int (*flush)(const DNNModel *model);
186  // Frees memory allocated for model.
187  void (*free_model)(DNNModel **model);
188 };
189 
190 // Initializes DNNModule depending on chosen backend.
191 const DNNModule *ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx);
192 
194 void *ff_dnn_child_next(DnnContext *obj, void *prev);
195 const AVClass *ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t backend_mask);
196 
198 {
199  return layout == DL_NHWC ? 2 : 3;
200 }
201 
203 {
204  return layout == DL_NHWC ? 1 : 2;
205 }
206 
208 {
209  return layout == DL_NHWC ? 3 : 1;
210 }
211 
212 #endif
DNNModule::type
DNNBackendType type
Definition: dnn_interface.h:177
TFOptions::sess_config
char * sess_config
Definition: dnn_interface.h:123
DNNColorOrder
DNNColorOrder
Definition: dnn_interface.h:43
out
FILE * out
Definition: movenc.c:55
DNNModule::get_result
DNNAsyncStatusType(* get_result)(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_interface.h:183
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:56
DnnContext::model
DNNModel * model
Definition: dnn_interface.h:146
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
DNNData::data
void * data
Definition: dnn_interface.h:70
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:374
OVOptions::mean
float mean
Definition: dnn_interface.h:133
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:110
DetectPostProc
int(* DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:94
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:81
DnnContext::clazz
const AVClass * clazz
Definition: dnn_interface.h:144
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:84
THOptions
Definition: dnn_interface.h:136
DFT_NONE
@ DFT_NONE
Definition: dnn_interface.h:57
OVOptions::batch_size
int batch_size
Definition: dnn_interface.h:129
DnnContext::dnn_module
const DNNModule * dnn_module
Definition: dnn_interface.h:157
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:99
dnn_get_width_idx_by_layout
static int dnn_get_width_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:197
DnnContext
Definition: dnn_interface.h:143
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:52
DAST_FAIL
@ DAST_FAIL
Definition: dnn_interface.h:50
DL_NHWC
@ DL_NHWC
Definition: dnn_interface.h:66
OVOptions::clazz
const AVClass * clazz
Definition: dnn_interface.h:127
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:36
DnnContext::async
int async
Definition: dnn_interface.h:153
DnnContext::model_filename
char * model_filename
Definition: dnn_interface.h:148
DCO_NONE
@ DCO_NONE
Definition: dnn_interface.h:44
DNNExecClassificationParams
Definition: dnn_interface.h:88
DnnContext::model_outputnames
char ** model_outputnames
Definition: dnn_interface.h:155
OVOptions::layout
DNNLayout layout
Definition: dnn_interface.h:131
DNNData::order
DNNColorOrder order
Definition: dnn_interface.h:74
DNNData
Definition: dnn_interface.h:69
DNNModule::clazz
const AVClass clazz
Definition: dnn_interface.h:176
DNNModel::get_output
int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:106
ctx
AVFormatContext * ctx
Definition: movenc.c:49
DL_NCHW
@ DL_NCHW
Definition: dnn_interface.h:65
DNN_OV
@ DNN_OV
Definition: dnn_interface.h:37
DNNExecClassificationParams::target
const char * target
Definition: dnn_interface.h:90
DnnContext::backend_options
char * backend_options
Definition: dnn_interface.h:152
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
DNNExecClassificationParams::base
DNNExecBaseParams base
Definition: dnn_interface.h:89
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:113
TFOptions::clazz
const AVClass * clazz
Definition: dnn_interface.h:121
ff_get_dnn_module
const DNNModule * ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
Definition: dnn_interface.c:82
DnnContext::nireq
int nireq
Definition: dnn_interface.h:159
DnnContext::backend_type
DNNBackendType backend_type
Definition: dnn_interface.h:149
ClassifyPostProc
int(* ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:95
DAST_SUCCESS
@ DAST_SUCCESS
Definition: dnn_interface.h:53
DnnContext::model_inputname
char * model_inputname
Definition: dnn_interface.h:150
DNNBackendType
DNNBackendType
Definition: dnn_interface.h:35
DAST_EMPTY_QUEUE
@ DAST_EMPTY_QUEUE
Definition: dnn_interface.h:51
DnnContext::nb_outputs
uint32_t nb_outputs
Definition: dnn_interface.h:156
DNNLayout
DNNLayout
Definition: dnn_interface.h:63
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:115
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:101
DNNDataType
DNNDataType
Definition: dnn_interface.h:41
DNNData::dt
DNNDataType dt
Definition: dnn_interface.h:73
frame.h
DNNData::scale
float scale
Definition: dnn_interface.h:76
DNNData::layout
DNNLayout layout
Definition: dnn_interface.h:75
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:41
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:85
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
OVOptions::input_resizable
int input_resizable
Definition: dnn_interface.h:130
THOptions::optimize
int optimize
Definition: dnn_interface.h:138
layout
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel layout
Definition: filter_design.txt:18
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:59
ff_dnn_child_next
void * ff_dnn_child_next(DnnContext *obj, void *prev)
Definition: dnn_interface.c:103
DnnContext::device
char * device
Definition: dnn_interface.h:160
DNNModel::classify_post_proc
ClassifyPostProc classify_post_proc
Definition: dnn_interface.h:117
OVOptions::scale
float scale
Definition: dnn_interface.h:132
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
DNNData::mean
float mean
Definition: dnn_interface.h:77
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:41
DFT_ANALYTICS_CLASSIFY
@ DFT_ANALYTICS_CLASSIFY
Definition: dnn_interface.h:60
DNNModule::free_model
void(* free_model)(DNNModel **model)
Definition: dnn_interface.h:187
avfilter.h
THOptions::clazz
const AVClass * clazz
Definition: dnn_interface.h:137
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:46
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:82
FramePrePostProc
int(* FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx)
Definition: dnn_interface.h:93
DL_NONE
@ DL_NONE
Definition: dnn_interface.h:64
AVFilterContext
An instance of a filter.
Definition: avfilter.h:407
DNNModel
Definition: dnn_interface.h:97
DNNData::dims
int dims[4]
Definition: dnn_interface.h:71
DNN_TH
@ DNN_TH
Definition: dnn_interface.h:38
dnn_get_height_idx_by_layout
static int dnn_get_height_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:202
ff_dnn_init_child_class
void ff_dnn_init_child_class(DnnContext *ctx)
Definition: dnn_interface.c:95
dnn_get_channel_idx_by_layout
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:207
ff_dnn_child_class_iterate_with_mask
const AVClass * ff_dnn_child_class_iterate_with_mask(void **iter, uint32_t backend_mask)
Definition: dnn_interface.c:123
TFOptions
Definition: dnn_interface.h:120
OVOptions
Definition: dnn_interface.h:126
DNNExecBaseParams
Definition: dnn_interface.h:80
DNNModel::get_input
int(* get_input)(struct DNNModel *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:104
DCO_BGR
@ DCO_BGR
Definition: dnn_interface.h:45
DAST_NOT_READY
@ DAST_NOT_READY
Definition: dnn_interface.h:52
int
int
Definition: ffmpeg_filter.c:424
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:49
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:58
DNNModule
Definition: dnn_interface.h:175
DNNModule::flush
int(* flush)(const DNNModel *model)
Definition: dnn_interface.h:185
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:83
DnnContext::model_outputnames_string
char * model_outputnames_string
Definition: dnn_interface.h:151
DNNModule::execute_model
int(* execute_model)(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_interface.h:181