FFmpeg
Loading...
Searching...
No Matches
dnn_filter_common.c
Go to the documentation of this file.
1/*
2 * This file is part of FFmpeg.
3 *
4 * FFmpeg is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU Lesser General Public
6 * License as published by the Free Software Foundation; either
7 * version 2.1 of the License, or (at your option) any later version.
8 *
9 * FFmpeg is distributed in the hope that it will be useful,
10 * but WITHOUT ANY WARRANTY; without even the implied warranty of
11 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
12 * Lesser General Public License for more details.
13 *
14 * You should have received a copy of the GNU Lesser General Public
15 * License along with FFmpeg; if not, write to the Free Software
16 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
17 */
18
19#include "config.h"
20#include "dnn_filter_common.h"
21#include "libavutil/avstring.h"
22#include "libavutil/mem.h"
23#include "libavutil/opt.h"
24#include "libavutil/hwcontext.h"
25#include "libavutil/pixdesc.h"
26
27#define MAX_SUPPORTED_OUTPUTS_NB 4
28
29static char **separate_output_names(const char *expr, const char *val_sep, int *separated_nb)
30{
31 char *val, **parsed_vals = NULL;
32 int val_num = 0;
33 if (!expr || !val_sep || !separated_nb) {
34 return NULL;
35 }
36
37 parsed_vals = av_calloc(MAX_SUPPORTED_OUTPUTS_NB + 1, sizeof(*parsed_vals));
38 if (!parsed_vals) {
39 return NULL;
40 }
41
42 do {
43 if (val_num >= MAX_SUPPORTED_OUTPUTS_NB) {
44 goto err;
45 }
46 val = av_get_token(&expr, val_sep);
47 if(val) {
48 parsed_vals[val_num] = val;
49 val_num++;
50 }
51 if (*expr) {
52 expr++;
53 }
54 } while(*expr);
55
56 parsed_vals[val_num] = NULL;
57 *separated_nb = val_num;
58
59 return parsed_vals;
60
61err:
62 for (int i = 0; i < val_num; i++)
63 av_free(parsed_vals[i]);
64 av_freep(&parsed_vals);
65 return NULL;
66}
67
68typedef struct DnnFilterBase {
69 const AVClass *class;
72
78
79void *ff_dnn_filter_child_next(void *obj, void *prev)
80{
82 return ff_dnn_child_next(&base->dnnctx, prev);
83}
84
86{
87 DNNBackendType backend = ctx->backend_type;
88
89 if (!ctx->model_filename) {
90 av_log(filter_ctx, AV_LOG_ERROR, "model file for network is not specified\n");
91 return AVERROR(EINVAL);
92 }
93
94 if (backend == DNN_TH) {
95 if (ctx->model_inputname)
96 av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require inputname, "\
97 "inputname will be ignored.\n");
98 if (ctx->model_outputnames)
99 av_log(filter_ctx, AV_LOG_WARNING, "LibTorch backend do not require outputname(s), "\
100 "all outputname(s) will be ignored.\n");
101 ctx->nb_outputs = 1;
102 } else if (backend == DNN_TF) {
103 if (!ctx->model_inputname) {
104 av_log(filter_ctx, AV_LOG_ERROR, "input name of the model network is not specified\n");
105 return AVERROR(EINVAL);
106 }
107 ctx->model_outputnames = separate_output_names(ctx->model_outputnames_string, "&", &ctx->nb_outputs);
108 if (!ctx->model_outputnames) {
109 av_log(filter_ctx, AV_LOG_ERROR, "could not parse model output names\n");
110 return AVERROR(EINVAL);
111 }
112 } else if (backend == DNN_ONNX) {
113 /* ONNX: input and output tensor names are optional.
114 * Multiple output names may be specified separated by '&'. */
115 if (ctx->model_outputnames_string) {
116 ctx->model_outputnames = separate_output_names(ctx->model_outputnames_string, "&", &ctx->nb_outputs);
117 if (!ctx->model_outputnames) {
118 av_log(filter_ctx, AV_LOG_ERROR, "could not parse model output names\n");
119 return AVERROR(EINVAL);
120 }
121 }
122 }
123
124 ctx->dnn_module = ff_get_dnn_module(ctx->backend_type, filter_ctx);
125 if (!ctx->dnn_module) {
126 av_log(filter_ctx, AV_LOG_ERROR, "could not create DNN module for requested backend\n");
127 return AVERROR(ENOMEM);
128 }
129 if (!ctx->dnn_module->load_model) {
130 av_log(filter_ctx, AV_LOG_ERROR, "load_model for network is not specified\n");
131 return AVERROR(EINVAL);
132 }
133
134 if (ctx->backend_options) {
135 void *child = NULL;
136
138 "backend_configs is deprecated, please set backend options directly\n");
139 while (child = ff_dnn_child_next(ctx, child)) {
140 if (*(const AVClass **)child == &ctx->dnn_module->clazz) {
141 int ret = av_opt_set_from_string(child, ctx->backend_options,
142 NULL, "=", "&");
143 if (ret < 0) {
144 av_log(filter_ctx, AV_LOG_ERROR, "failed to parse options \"%s\"\n",
145 ctx->backend_options);
146 return ret;
147 }
148 }
149 }
150 }
151
152 ctx->model = (ctx->dnn_module->load_model)(ctx, func_type, filter_ctx);
153 if (!ctx->model) {
154 av_log(filter_ctx, AV_LOG_ERROR, "could not load DNN model\n");
155 return AVERROR(EINVAL);
156 }
157
158 return 0;
159}
160
162{
163 ctx->model->frame_pre_proc = pre_proc;
164 ctx->model->frame_post_proc = post_proc;
165 return 0;
166}
167
169{
170 ctx->model->detect_post_proc = post_proc;
171 return 0;
172}
173
175{
176 ctx->model->classify_post_proc = post_proc;
177 return 0;
178}
179
181{
182 return ctx->model->get_input(ctx->model, input, ctx->model_inputname);
183}
184
185int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height)
186{
187 char * output_name = ctx->model_outputnames && ctx->backend_type != DNN_TH ?
188 ctx->model_outputnames[0] : NULL;
189 return ctx->model->get_output(ctx->model, ctx->model_inputname, input_width, input_height,
190 (const char *)output_name, output_width, output_height);
191}
192
194{
195 av_buffer_unref(&ctx->hw_frames_ctx);
196 if (hw_frames_ctx) {
197 ctx->hw_frames_ctx = av_buffer_ref(hw_frames_ctx);
198 if (!ctx->hw_frames_ctx)
199 return AVERROR(ENOMEM);
200 }
201 return 0;
202}
203
205{
206 DNNExecBaseParams exec_params = {
207 .input_name = ctx->model_inputname,
208 .output_names = (const char **)ctx->model_outputnames,
209 .nb_output = ctx->nb_outputs,
210 .in_frame = in_frame,
211 .out_frame = out_frame,
212 };
213 return (ctx->dnn_module->execute_model)(ctx->model, &exec_params);
214}
215
216int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char *target)
217{
218 DNNExecClassificationParams class_params = {
219 {
220 .input_name = ctx->model_inputname,
221 .output_names = (const char **)ctx->model_outputnames,
222 .nb_output = ctx->nb_outputs,
223 .in_frame = in_frame,
224 .out_frame = out_frame,
225 },
226 .target = target,
227 };
228 return (ctx->dnn_module->execute_model)(ctx->model, &class_params.base);
229}
230
232{
233 return (ctx->dnn_module->get_result)(ctx->model, in_frame, out_frame);
234}
235
237{
238 return (ctx->dnn_module->flush)(ctx->model);
239}
240
242{
243 if (ctx->dnn_module) {
244 (ctx->dnn_module->free_model)(&ctx->model);
245 }
246 for (void *child = NULL; (child = ff_dnn_child_next(ctx, child));)
247 av_opt_free(child);
248 if (ctx->model_outputnames) {
249 for (int i = 0; i < ctx->nb_outputs; i++)
250 av_free(ctx->model_outputnames[i]);
251
252 av_freep(&ctx->model_outputnames);
253 }
254 av_buffer_unref(&ctx->hw_frames_ctx);
255}
256
257#if CONFIG_CUDA
258int ff_dnn_zero_copy_supported_cuda(DnnContext *ctx, const AVFilterLink *inlink)
259{
260 AVBufferRef *hw_frames_ref = avfilter_link_get_hw_frames_ctx((AVFilterLink *)inlink);
261 AVHWFramesContext *hw_frames_ctx;
262
263 if (!hw_frames_ref)
264 return 0;
265
266 hw_frames_ctx = (AVHWFramesContext *)hw_frames_ref->data;
267
268 if (inlink->format == AV_PIX_FMT_CUDA) {
269 if (ctx->batch_size > 1) {
270 av_log(inlink->dst, AV_LOG_ERROR, "CUDA zero-copy currently does not support batching.\n");
271 av_buffer_unref(&hw_frames_ref);
272 return AVERROR(EINVAL);
273 }
274
275 if (ctx->backend_type == DNN_TH) {
276 switch (hw_frames_ctx->sw_format) {
277 case AV_PIX_FMT_RGB24:
278 case AV_PIX_FMT_BGR24:
279 case AV_PIX_FMT_RGB0:
280 case AV_PIX_FMT_0RGB:
281 case AV_PIX_FMT_BGR0:
282 case AV_PIX_FMT_0BGR:
283 break;
284 default:
285 av_log(inlink->dst, AV_LOG_ERROR,
286 "Zero-copy CUDA path currently only supports RGB24/BGR24 or RGB0/BGR0 variants.\n");
287 av_buffer_unref(&hw_frames_ref);
288 return AVERROR(EINVAL);
289 }
290 }
291 }
292
293 av_buffer_unref(&hw_frames_ref);
294 return 0;
295}
296#endif
297
298#if CONFIG_DNN_ONNX_D3D12
299int ff_dnn_zero_copy_supported_d3d12(DnnContext *ctx, const AVFilterLink *inlink)
300{
301 AVBufferRef *hw_frames_ref;
302 AVHWFramesContext *hw_frames_ctx;
303
304 if (ctx->backend_type != DNN_ONNX) {
305 av_log(inlink->dst, AV_LOG_ERROR,
306 "D3D12 zero-copy requires the ONNX Runtime backend (dnn_backend=onnx).\n");
307 return AVERROR(EINVAL);
308 }
309
310 if (ctx->batch_size > 1) {
311 av_log(inlink->dst, AV_LOG_ERROR, "D3D12 zero-copy currently does not support batching.\n");
312 return AVERROR(EINVAL);
313 }
314
315 hw_frames_ref = avfilter_link_get_hw_frames_ctx((AVFilterLink *)inlink);
316 if (!hw_frames_ref) {
317 av_log(inlink->dst, AV_LOG_ERROR, "D3D12 input has no hw_frames_ctx.\n");
318 return AVERROR(EINVAL);
319 }
320 hw_frames_ctx = (AVHWFramesContext *)hw_frames_ref->data;
321
322 if (hw_frames_ctx->sw_format != AV_PIX_FMT_NV12) {
323 av_log(inlink->dst, AV_LOG_ERROR,
324 "D3D12 zero-copy currently only supports NV12 hardware frames, got %s.\n",
325 av_get_pix_fmt_name(hw_frames_ctx->sw_format));
326 av_buffer_unref(&hw_frames_ref);
327 return AVERROR(EINVAL);
328 }
329
330 av_buffer_unref(&hw_frames_ref);
331
332 if (!ctx->zero_copy_negotiated) {
333 av_log(inlink->dst, AV_LOG_ERROR,
334 "D3D12 zero-copy was not negotiated by the ONNX Runtime backend. "
335 "Check that device=dml is set, that this build's ONNX Runtime "
336 "provides a working DirectML execution provider, and that "
337 "DirectML.dll and d3dcompiler_47.dll are available in the "
338 "application directory or System32.\n");
339 return AVERROR(ENOSYS);
340 }
341
342 return 0;
343}
344#endif
static double val(void *priv, double ch)
Definition aeval.c:77
static AVFormatContext * ctx
#define i(width, name, range_min, range_max)
Definition cbs_h264.c:63
#define NULL
Definition coverity.c:32
static char ** separate_output_names(const char *expr, const char *val_sep, int *separated_nb)
void * ff_dnn_filter_child_next(void *obj, void *prev)
int ff_dnn_set_detect_post_proc(DnnContext *ctx, DetectPostProc post_proc)
void ff_dnn_uninit(DnnContext *ctx)
DNNAsyncStatusType ff_dnn_get_result(DnnContext *ctx, AVFrame **in_frame, AVFrame **out_frame)
int ff_dnn_execute_model(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame)
int ff_dnn_set_classify_post_proc(DnnContext *ctx, ClassifyPostProc post_proc)
int ff_dnn_init(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
int ff_dnn_set_hw_frames_ctx(DnnContext *ctx, AVBufferRef *hw_frames_ctx)
int ff_dnn_get_input(DnnContext *ctx, DNNData *input)
#define MAX_SUPPORTED_OUTPUTS_NB
int ff_dnn_get_output(DnnContext *ctx, int input_width, int input_height, int *output_width, int *output_height)
int ff_dnn_flush(DnnContext *ctx)
int ff_dnn_set_frame_proc(DnnContext *ctx, FramePrePostProc pre_proc, FramePrePostProc post_proc)
int ff_dnn_filter_init_child_class(AVFilterContext *filter)
int ff_dnn_execute_model_classification(DnnContext *ctx, AVFrame *in_frame, AVFrame *out_frame, const char *target)
common functions for the dnn based filters
void * ff_dnn_child_next(DnnContext *obj, void *prev)
const DNNModule * ff_get_dnn_module(DNNBackendType backend_type, void *log_ctx)
void ff_dnn_init_child_class(DnnContext *ctx)
int(* FramePrePostProc)(AVFrame *frame, DNNData *model, AVFilterContext *filter_ctx)
DNNAsyncStatusType
int(* DetectPostProc)(AVFrame *frame, DNNData *output, uint32_t nb, AVFilterContext *filter_ctx)
DNNBackendType
@ DNN_ONNX
@ DNN_TH
@ DNN_TF
DNNFunctionType
int(* ClassifyPostProc)(AVFrame *frame, DNNData *output, uint32_t bbox_index, AVFilterContext *filter_ctx)
int ret
Definition ffplay.c:391
AVBufferRef * avfilter_link_get_hw_frames_ctx(AVFilterLink *link)
Get the hardware frames context of a filter link.
Definition avfilter.c:1009
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it.
Definition buffer.c:140
AVBufferRef * av_buffer_ref(const AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition buffer.c:104
#define AVERROR(e)
Definition error.h:45
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition log.h:216
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition log.h:210
char * av_get_token(const char **buf, const char *term)
Unescape the given string until a non escaped terminating char, and return the token corresponding to...
Definition avstring.c:143
void av_opt_free(void *obj)
Free all allocated objects in obj.
Definition opt.c:2042
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition opt.c:1990
void * av_calloc(size_t nmemb, size_t size)
Definition mem.c:370
Memory handling functions.
static const char * obj
Definition mscl.c:57
AVOptions.
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition pixdesc.c:3404
@ AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition pixfmt.h:96
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
Definition pixfmt.h:75
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
Definition pixfmt.h:265
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
Definition pixfmt.h:260
@ AV_PIX_FMT_0BGR
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
Definition pixfmt.h:264
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
Definition pixfmt.h:263
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
Definition pixfmt.h:76
@ AV_PIX_FMT_0RGB
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
Definition pixfmt.h:262
A reference to a data buffer.
Definition buffer.h:82
uint8_t * data
The data buffer.
Definition buffer.h:90
Describe the class of an AVClass context structure.
Definition log.h:76
An instance of a filter.
Definition avfilter.h:279
This structure describes decoded (raw) audio or video data.
Definition frame.h:479
This struct describes a set or pool of "hardware" frames (i.e.
Definition hwcontext.h:118
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
Definition hwcontext.h:213
#define av_free(p)
#define av_freep(p)
#define av_log(a,...)
void(* filter)(uint8_t *src, ptrdiff_t stride, int qscale)
Definition h263dsp.c:29
static FilteringContext * filter_ctx
Definition transcode.c:52
uint8_t base
Definition vp3data.h:128