FFmpeg
dnn_backend_openvino.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN OpenVINO backend implementation.
24  */
25 
26 #include "dnn_backend_openvino.h"
27 #include "dnn_io_proc.h"
28 #include "libavformat/avio.h"
29 #include "libavutil/avassert.h"
30 #include "libavutil/opt.h"
31 #include "libavutil/avstring.h"
32 #include "../internal.h"
33 #include "queue.h"
34 #include "safe_queue.h"
35 #include <c_api/ie_c_api.h>
36 
37 typedef struct OVOptions{
38  char *device_type;
39  int nireq;
42 } OVOptions;
43 
44 typedef struct OVContext {
45  const AVClass *class;
47 } OVContext;
48 
49 typedef struct OVModel{
52  ie_core_t *core;
53  ie_network_t *network;
54  ie_executable_network_t *exe_network;
55  ie_infer_request_t *infer_request;
56 
57  /* for async execution */
58  SafeQueue *request_queue; // holds RequestItem
59  Queue *task_queue; // holds TaskItem
60 } OVModel;
61 
62 typedef struct TaskItem {
64  const char *input_name;
66  const char *output_name;
68  int do_ioproc;
69  int async;
70  int done;
71 } TaskItem;
72 
73 typedef struct RequestItem {
74  ie_infer_request_t *infer_request;
77  ie_complete_call_back_t callback;
78 } RequestItem;
79 
80 #define APPEND_STRING(generated_string, iterate_string) \
81  generated_string = generated_string ? av_asprintf("%s %s", generated_string, iterate_string) : \
82  av_asprintf("%s", iterate_string);
83 
84 #define OFFSET(x) offsetof(OVContext, x)
85 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
86 static const AVOption dnn_openvino_options[] = {
87  { "device", "device to run model", OFFSET(options.device_type), AV_OPT_TYPE_STRING, { .str = "CPU" }, 0, 0, FLAGS },
88  { "nireq", "number of request", OFFSET(options.nireq), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS },
89  { "batch_size", "batch size per request", OFFSET(options.batch_size), AV_OPT_TYPE_INT, { .i64 = 1 }, 1, 1000, FLAGS},
90  { "input_resizable", "can input be resizable or not", OFFSET(options.input_resizable), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, FLAGS },
91  { NULL }
92 };
93 
94 AVFILTER_DEFINE_CLASS(dnn_openvino);
95 
96 static DNNDataType precision_to_datatype(precision_e precision)
97 {
98  switch (precision)
99  {
100  case FP32:
101  return DNN_FLOAT;
102  case U8:
103  return DNN_UINT8;
104  default:
105  av_assert0(!"not supported yet.");
106  return DNN_FLOAT;
107  }
108 }
109 
111 {
112  switch (dt)
113  {
114  case DNN_FLOAT:
115  return sizeof(float);
116  case DNN_UINT8:
117  return sizeof(uint8_t);
118  default:
119  av_assert0(!"not supported yet.");
120  return 1;
121  }
122 }
123 
125 {
126  dimensions_t dims;
127  precision_e precision;
128  ie_blob_buffer_t blob_buffer;
129  OVContext *ctx = &ov_model->ctx;
130  IEStatusCode status;
131  DNNData input;
132  ie_blob_t *input_blob = NULL;
133  TaskItem *task = request->tasks[0];
134 
135  status = ie_infer_request_get_blob(request->infer_request, task->input_name, &input_blob);
136  if (status != OK) {
137  av_log(ctx, AV_LOG_ERROR, "Failed to get input blob with name %s\n", task->input_name);
138  return DNN_ERROR;
139  }
140 
141  status |= ie_blob_get_dims(input_blob, &dims);
142  status |= ie_blob_get_precision(input_blob, &precision);
143  if (status != OK) {
144  ie_blob_free(&input_blob);
145  av_log(ctx, AV_LOG_ERROR, "Failed to get input blob dims/precision\n");
146  return DNN_ERROR;
147  }
148 
149  status = ie_blob_get_buffer(input_blob, &blob_buffer);
150  if (status != OK) {
151  ie_blob_free(&input_blob);
152  av_log(ctx, AV_LOG_ERROR, "Failed to get input blob buffer\n");
153  return DNN_ERROR;
154  }
155 
156  input.height = dims.dims[2];
157  input.width = dims.dims[3];
158  input.channels = dims.dims[1];
159  input.data = blob_buffer.buffer;
160  input.dt = precision_to_datatype(precision);
161  // all models in openvino open model zoo use BGR as input,
162  // change to be an option when necessary.
163  input.order = DCO_BGR;
164 
165  av_assert0(request->task_count <= dims.dims[0]);
166  for (int i = 0; i < request->task_count; ++i) {
167  task = request->tasks[i];
168  if (task->do_ioproc) {
169  if (ov_model->model->frame_pre_proc != NULL) {
170  ov_model->model->frame_pre_proc(task->in_frame, &input, ov_model->model->filter_ctx);
171  } else {
172  ff_proc_from_frame_to_dnn(task->in_frame, &input, ov_model->model->func_type, ctx);
173  }
174  }
175  input.data = (uint8_t *)input.data
176  + input.width * input.height * input.channels * get_datatype_size(input.dt);
177  }
178  ie_blob_free(&input_blob);
179 
180  return DNN_SUCCESS;
181 }
182 
183 static void infer_completion_callback(void *args)
184 {
185  dimensions_t dims;
186  precision_e precision;
187  IEStatusCode status;
188  RequestItem *request = args;
189  TaskItem *task = request->tasks[0];
190  SafeQueue *requestq = task->ov_model->request_queue;
191  ie_blob_t *output_blob = NULL;
192  ie_blob_buffer_t blob_buffer;
193  DNNData output;
194  OVContext *ctx = &task->ov_model->ctx;
195 
196  status = ie_infer_request_get_blob(request->infer_request, task->output_name, &output_blob);
197  if (status != OK) {
198  //incorrect output name
199  char *model_output_name = NULL;
200  char *all_output_names = NULL;
201  size_t model_output_count = 0;
202  av_log(ctx, AV_LOG_ERROR, "Failed to get model output data\n");
203  status = ie_network_get_outputs_number(task->ov_model->network, &model_output_count);
204  for (size_t i = 0; i < model_output_count; i++) {
205  status = ie_network_get_output_name(task->ov_model->network, i, &model_output_name);
206  APPEND_STRING(all_output_names, model_output_name)
207  }
208  av_log(ctx, AV_LOG_ERROR,
209  "output \"%s\" may not correct, all output(s) are: \"%s\"\n",
210  task->output_name, all_output_names);
211  return;
212  }
213 
214  status = ie_blob_get_buffer(output_blob, &blob_buffer);
215  if (status != OK) {
216  ie_blob_free(&output_blob);
217  av_log(ctx, AV_LOG_ERROR, "Failed to access output memory\n");
218  return;
219  }
220 
221  status |= ie_blob_get_dims(output_blob, &dims);
222  status |= ie_blob_get_precision(output_blob, &precision);
223  if (status != OK) {
224  ie_blob_free(&output_blob);
225  av_log(ctx, AV_LOG_ERROR, "Failed to get dims or precision of output\n");
226  return;
227  }
228 
229  output.channels = dims.dims[1];
230  output.height = dims.dims[2];
231  output.width = dims.dims[3];
232  output.dt = precision_to_datatype(precision);
233  output.data = blob_buffer.buffer;
234 
235  av_assert0(request->task_count <= dims.dims[0]);
236  av_assert0(request->task_count >= 1);
237  for (int i = 0; i < request->task_count; ++i) {
238  task = request->tasks[i];
239 
240  switch (task->ov_model->model->func_type) {
241  case DFT_PROCESS_FRAME:
242  if (task->do_ioproc) {
243  if (task->ov_model->model->frame_post_proc != NULL) {
244  task->ov_model->model->frame_post_proc(task->out_frame, &output, task->ov_model->model->filter_ctx);
245  } else {
246  ff_proc_from_dnn_to_frame(task->out_frame, &output, ctx);
247  }
248  } else {
249  task->out_frame->width = output.width;
250  task->out_frame->height = output.height;
251  }
252  break;
254  if (!task->ov_model->model->detect_post_proc) {
255  av_log(ctx, AV_LOG_ERROR, "detect filter needs to provide post proc\n");
256  return;
257  }
258  task->ov_model->model->detect_post_proc(task->out_frame, &output, 1, task->ov_model->model->filter_ctx);
259  break;
260  default:
261  av_assert0(!"should not reach here");
262  break;
263  }
264 
265  task->done = 1;
266  output.data = (uint8_t *)output.data
267  + output.width * output.height * output.channels * get_datatype_size(output.dt);
268  }
269  ie_blob_free(&output_blob);
270 
271  request->task_count = 0;
272 
273  if (task->async) {
274  if (ff_safe_queue_push_back(requestq, request) < 0) {
275  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
276  return;
277  }
278  }
279 }
280 
281 static DNNReturnType init_model_ov(OVModel *ov_model, const char *input_name, const char *output_name)
282 {
283  OVContext *ctx = &ov_model->ctx;
284  IEStatusCode status;
285  ie_available_devices_t a_dev;
286  ie_config_t config = {NULL, NULL, NULL};
287  char *all_dev_names = NULL;
288 
289  // batch size
290  if (ctx->options.batch_size <= 0) {
291  ctx->options.batch_size = 1;
292  }
293 
294  if (ctx->options.batch_size > 1) {
295  input_shapes_t input_shapes;
296  status = ie_network_get_input_shapes(ov_model->network, &input_shapes);
297  if (status != OK)
298  goto err;
299  for (int i = 0; i < input_shapes.shape_num; i++)
300  input_shapes.shapes[i].shape.dims[0] = ctx->options.batch_size;
301  status = ie_network_reshape(ov_model->network, input_shapes);
302  ie_network_input_shapes_free(&input_shapes);
303  if (status != OK)
304  goto err;
305  }
306 
307  // The order of dims in the openvino is fixed and it is always NCHW for 4-D data.
308  // while we pass NHWC data from FFmpeg to openvino
309  status = ie_network_set_input_layout(ov_model->network, input_name, NHWC);
310  if (status != OK) {
311  av_log(ctx, AV_LOG_ERROR, "Failed to set layout as NHWC for input %s\n", input_name);
312  goto err;
313  }
314  status = ie_network_set_output_layout(ov_model->network, output_name, NHWC);
315  if (status != OK) {
316  av_log(ctx, AV_LOG_ERROR, "Failed to set layout as NHWC for output %s\n", output_name);
317  goto err;
318  }
319 
320  // all models in openvino open model zoo use BGR with range [0.0f, 255.0f] as input,
321  // we don't have a AVPixelFormat to descibe it, so we'll use AV_PIX_FMT_BGR24 and
322  // ask openvino to do the conversion internally.
323  // the current supported SR model (frame processing) is generated from tensorflow model,
324  // and its input is Y channel as float with range [0.0f, 1.0f], so do not set for this case.
325  // TODO: we need to get a final clear&general solution with all backends/formats considered.
326  if (ov_model->model->func_type != DFT_PROCESS_FRAME) {
327  status = ie_network_set_input_precision(ov_model->network, input_name, U8);
328  if (status != OK) {
329  av_log(ctx, AV_LOG_ERROR, "Failed to set input precision as U8 for %s\n", input_name);
330  goto err;
331  }
332  }
333 
334  status = ie_core_load_network(ov_model->core, ov_model->network, ctx->options.device_type, &config, &ov_model->exe_network);
335  if (status != OK) {
336  av_log(ctx, AV_LOG_ERROR, "Failed to load OpenVINO model network\n");
337  status = ie_core_get_available_devices(ov_model->core, &a_dev);
338  if (status != OK) {
339  av_log(ctx, AV_LOG_ERROR, "Failed to get available devices\n");
340  goto err;
341  }
342  for (int i = 0; i < a_dev.num_devices; i++) {
343  APPEND_STRING(all_dev_names, a_dev.devices[i])
344  }
345  av_log(ctx, AV_LOG_ERROR,"device %s may not be supported, all available devices are: \"%s\"\n",
346  ctx->options.device_type, all_dev_names);
347  goto err;
348  }
349 
350  // create infer_request for sync execution
351  status = ie_exec_network_create_infer_request(ov_model->exe_network, &ov_model->infer_request);
352  if (status != OK)
353  goto err;
354 
355  // create infer_requests for async execution
356  if (ctx->options.nireq <= 0) {
357  // the default value is a rough estimation
358  ctx->options.nireq = av_cpu_count() / 2 + 1;
359  }
360 
361  ov_model->request_queue = ff_safe_queue_create();
362  if (!ov_model->request_queue) {
363  goto err;
364  }
365 
366  for (int i = 0; i < ctx->options.nireq; i++) {
367  RequestItem *item = av_mallocz(sizeof(*item));
368  if (!item) {
369  goto err;
370  }
371 
372  item->callback.completeCallBackFunc = infer_completion_callback;
373  item->callback.args = item;
374  if (ff_safe_queue_push_back(ov_model->request_queue, item) < 0) {
375  av_freep(&item);
376  goto err;
377  }
378 
379  status = ie_exec_network_create_infer_request(ov_model->exe_network, &item->infer_request);
380  if (status != OK) {
381  goto err;
382  }
383 
384  item->tasks = av_malloc_array(ctx->options.batch_size, sizeof(*item->tasks));
385  if (!item->tasks) {
386  goto err;
387  }
388  item->task_count = 0;
389  }
390 
391  ov_model->task_queue = ff_queue_create();
392  if (!ov_model->task_queue) {
393  goto err;
394  }
395 
396  return DNN_SUCCESS;
397 
398 err:
399  ff_dnn_free_model_ov(&ov_model->model);
400  return DNN_ERROR;
401 }
402 
404 {
405  IEStatusCode status;
407  TaskItem *task = request->tasks[0];
408  OVContext *ctx = &task->ov_model->ctx;
409 
410  if (task->async) {
411  if (request->task_count < ctx->options.batch_size) {
412  if (ff_safe_queue_push_front(task->ov_model->request_queue, request) < 0) {
413  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
414  return DNN_ERROR;
415  }
416  return DNN_SUCCESS;
417  }
418  ret = fill_model_input_ov(task->ov_model, request);
419  if (ret != DNN_SUCCESS) {
420  return ret;
421  }
422  status = ie_infer_set_completion_callback(request->infer_request, &request->callback);
423  if (status != OK) {
424  av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
425  return DNN_ERROR;
426  }
427  status = ie_infer_request_infer_async(request->infer_request);
428  if (status != OK) {
429  av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n");
430  return DNN_ERROR;
431  }
432  return DNN_SUCCESS;
433  } else {
434  ret = fill_model_input_ov(task->ov_model, request);
435  if (ret != DNN_SUCCESS) {
436  return ret;
437  }
438  status = ie_infer_request_infer(request->infer_request);
439  if (status != OK) {
440  av_log(ctx, AV_LOG_ERROR, "Failed to start synchronous model inference\n");
441  return DNN_ERROR;
442  }
443  infer_completion_callback(request);
444  return task->done ? DNN_SUCCESS : DNN_ERROR;
445  }
446 }
447 
448 static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name)
449 {
450  OVModel *ov_model = model;
451  OVContext *ctx = &ov_model->ctx;
452  char *model_input_name = NULL;
453  char *all_input_names = NULL;
454  IEStatusCode status;
455  size_t model_input_count = 0;
456  dimensions_t dims;
457  precision_e precision;
459 
460  status = ie_network_get_inputs_number(ov_model->network, &model_input_count);
461  if (status != OK) {
462  av_log(ctx, AV_LOG_ERROR, "Failed to get input count\n");
463  return DNN_ERROR;
464  }
465 
466  for (size_t i = 0; i < model_input_count; i++) {
467  status = ie_network_get_input_name(ov_model->network, i, &model_input_name);
468  if (status != OK) {
469  av_log(ctx, AV_LOG_ERROR, "Failed to get No.%d input's name\n", (int)i);
470  return DNN_ERROR;
471  }
472  if (strcmp(model_input_name, input_name) == 0) {
473  ie_network_name_free(&model_input_name);
474  status |= ie_network_get_input_dims(ov_model->network, input_name, &dims);
475  status |= ie_network_get_input_precision(ov_model->network, input_name, &precision);
476  if (status != OK) {
477  av_log(ctx, AV_LOG_ERROR, "Failed to get No.%d input's dims or precision\n", (int)i);
478  return DNN_ERROR;
479  }
480 
481  input->channels = dims.dims[1];
482  input->height = input_resizable ? -1 : dims.dims[2];
483  input->width = input_resizable ? -1 : dims.dims[3];
484  input->dt = precision_to_datatype(precision);
485  return DNN_SUCCESS;
486  } else {
487  //incorrect input name
488  APPEND_STRING(all_input_names, model_input_name)
489  }
490 
491  ie_network_name_free(&model_input_name);
492  }
493 
494  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model, all input(s) are: \"%s\"\n", input_name, all_input_names);
495  return DNN_ERROR;
496 }
497 
498 static DNNReturnType get_output_ov(void *model, const char *input_name, int input_width, int input_height,
499  const char *output_name, int *output_width, int *output_height)
500 {
502  OVModel *ov_model = model;
503  OVContext *ctx = &ov_model->ctx;
504  TaskItem task;
505  RequestItem request;
506  AVFrame *in_frame = NULL;
507  AVFrame *out_frame = NULL;
508  TaskItem *ptask = &task;
509  IEStatusCode status;
510  input_shapes_t input_shapes;
511 
512  if (ov_model->model->func_type != DFT_PROCESS_FRAME) {
513  av_log(ctx, AV_LOG_ERROR, "Get output dim only when processing frame.\n");
514  return DNN_ERROR;
515  }
516 
517  if (ctx->options.input_resizable) {
518  status = ie_network_get_input_shapes(ov_model->network, &input_shapes);
519  input_shapes.shapes->shape.dims[2] = input_height;
520  input_shapes.shapes->shape.dims[3] = input_width;
521  status |= ie_network_reshape(ov_model->network, input_shapes);
522  ie_network_input_shapes_free(&input_shapes);
523  if (status != OK) {
524  av_log(ctx, AV_LOG_ERROR, "Failed to reshape input size for %s\n", input_name);
525  return DNN_ERROR;
526  }
527  }
528 
529  if (!ov_model->exe_network) {
530  if (init_model_ov(ov_model, input_name, output_name) != DNN_SUCCESS) {
531  av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
532  return DNN_ERROR;
533  }
534  }
535 
536  in_frame = av_frame_alloc();
537  if (!in_frame) {
538  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n");
539  return DNN_ERROR;
540  }
541  in_frame->width = input_width;
542  in_frame->height = input_height;
543 
544  out_frame = av_frame_alloc();
545  if (!out_frame) {
546  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output frame\n");
547  av_frame_free(&in_frame);
548  return DNN_ERROR;
549  }
550 
551  task.done = 0;
552  task.do_ioproc = 0;
553  task.async = 0;
554  task.input_name = input_name;
555  task.in_frame = in_frame;
556  task.output_name = output_name;
557  task.out_frame = out_frame;
558  task.ov_model = ov_model;
559 
560  request.infer_request = ov_model->infer_request;
561  request.task_count = 1;
562  request.tasks = &ptask;
563 
564  ret = execute_model_ov(&request);
565  *output_width = out_frame->width;
566  *output_height = out_frame->height;
567 
568  av_frame_free(&out_frame);
569  av_frame_free(&in_frame);
570  return ret;
571 }
572 
573 DNNModel *ff_dnn_load_model_ov(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
574 {
575  DNNModel *model = NULL;
576  OVModel *ov_model = NULL;
577  OVContext *ctx = NULL;
578  IEStatusCode status;
579 
580  model = av_mallocz(sizeof(DNNModel));
581  if (!model){
582  return NULL;
583  }
584 
585  ov_model = av_mallocz(sizeof(OVModel));
586  if (!ov_model) {
587  av_freep(&model);
588  return NULL;
589  }
590  model->model = ov_model;
591  ov_model->model = model;
592  ov_model->ctx.class = &dnn_openvino_class;
593  ctx = &ov_model->ctx;
594 
595  //parse options
596  av_opt_set_defaults(ctx);
597  if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {
598  av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
599  goto err;
600  }
601 
602  status = ie_core_create("", &ov_model->core);
603  if (status != OK)
604  goto err;
605 
606  status = ie_core_read_network(ov_model->core, model_filename, NULL, &ov_model->network);
607  if (status != OK) {
608  ie_version_t ver;
609  ver = ie_c_api_version();
610  av_log(ctx, AV_LOG_ERROR, "Failed to read the network from model file %s,\n"
611  "Please check if the model version matches the runtime OpenVINO %s\n",
612  model_filename, ver.api_version);
613  ie_version_free(&ver);
614  goto err;
615  }
616 
617  model->get_input = &get_input_ov;
618  model->get_output = &get_output_ov;
619  model->options = options;
620  model->filter_ctx = filter_ctx;
621  model->func_type = func_type;
622 
623  return model;
624 
625 err:
626  ff_dnn_free_model_ov(&model);
627  return NULL;
628 }
629 
630 DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
631  const char **output_names, uint32_t nb_output, AVFrame *out_frame)
632 {
633  OVModel *ov_model = model->model;
634  OVContext *ctx = &ov_model->ctx;
635  TaskItem task;
636  RequestItem request;
637  TaskItem *ptask = &task;
638 
639  if (!in_frame) {
640  av_log(ctx, AV_LOG_ERROR, "in frame is NULL when execute model.\n");
641  return DNN_ERROR;
642  }
643 
644  if (!out_frame && model->func_type == DFT_PROCESS_FRAME) {
645  av_log(ctx, AV_LOG_ERROR, "out frame is NULL when execute model.\n");
646  return DNN_ERROR;
647  }
648 
649  if (nb_output != 1) {
650  // currently, the filter does not need multiple outputs,
651  // so we just pending the support until we really need it.
652  avpriv_report_missing_feature(ctx, "multiple outputs");
653  return DNN_ERROR;
654  }
655 
656  if (ctx->options.batch_size > 1) {
657  avpriv_report_missing_feature(ctx, "batch mode for sync execution");
658  return DNN_ERROR;
659  }
660 
661  if (!ov_model->exe_network) {
662  if (init_model_ov(ov_model, input_name, output_names[0]) != DNN_SUCCESS) {
663  av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
664  return DNN_ERROR;
665  }
666  }
667 
668  task.done = 0;
669  task.do_ioproc = 1;
670  task.async = 0;
671  task.input_name = input_name;
672  task.in_frame = in_frame;
673  task.output_name = output_names[0];
674  task.out_frame = out_frame;
675  task.ov_model = ov_model;
676 
677  request.infer_request = ov_model->infer_request;
678  request.task_count = 1;
679  request.tasks = &ptask;
680 
681  return execute_model_ov(&request);
682 }
683 
684 DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame,
685  const char **output_names, uint32_t nb_output, AVFrame *out_frame)
686 {
687  OVModel *ov_model = model->model;
688  OVContext *ctx = &ov_model->ctx;
689  RequestItem *request;
690  TaskItem *task;
691 
692  if (!in_frame) {
693  av_log(ctx, AV_LOG_ERROR, "in frame is NULL when async execute model.\n");
694  return DNN_ERROR;
695  }
696 
697  if (!out_frame && model->func_type == DFT_PROCESS_FRAME) {
698  av_log(ctx, AV_LOG_ERROR, "out frame is NULL when async execute model.\n");
699  return DNN_ERROR;
700  }
701 
702  if (!ov_model->exe_network) {
703  if (init_model_ov(ov_model, input_name, output_names[0]) != DNN_SUCCESS) {
704  av_log(ctx, AV_LOG_ERROR, "Failed init OpenVINO exectuable network or inference request\n");
705  return DNN_ERROR;
706  }
707  }
708 
709  task = av_malloc(sizeof(*task));
710  if (!task) {
711  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
712  return DNN_ERROR;
713  }
714 
715  task->done = 0;
716  task->do_ioproc = 1;
717  task->async = 1;
718  task->input_name = input_name;
719  task->in_frame = in_frame;
720  task->output_name = output_names[0];
721  task->out_frame = out_frame;
722  task->ov_model = ov_model;
723  if (ff_queue_push_back(ov_model->task_queue, task) < 0) {
724  av_freep(&task);
725  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
726  return DNN_ERROR;
727  }
728 
729  request = ff_safe_queue_pop_front(ov_model->request_queue);
730  if (!request) {
731  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
732  return DNN_ERROR;
733  }
734 
735  request->tasks[request->task_count++] = task;
736  return execute_model_ov(request);
737 }
738 
740 {
741  OVModel *ov_model = model->model;
742  TaskItem *task = ff_queue_peek_front(ov_model->task_queue);
743 
744  if (!task) {
745  return DAST_EMPTY_QUEUE;
746  }
747 
748  if (!task->done) {
749  return DAST_NOT_READY;
750  }
751 
752  *in = task->in_frame;
753  *out = task->out_frame;
754  ff_queue_pop_front(ov_model->task_queue);
755  av_freep(&task);
756 
757  return DAST_SUCCESS;
758 }
759 
761 {
762  OVModel *ov_model = model->model;
763  OVContext *ctx = &ov_model->ctx;
764  RequestItem *request;
765  IEStatusCode status;
767 
768  request = ff_safe_queue_pop_front(ov_model->request_queue);
769  if (!request) {
770  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
771  return DNN_ERROR;
772  }
773 
774  if (request->task_count == 0) {
775  // no pending task need to flush
776  if (ff_safe_queue_push_back(ov_model->request_queue, request) < 0) {
777  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
778  return DNN_ERROR;
779  }
780  return DNN_SUCCESS;
781  }
782 
783  ret = fill_model_input_ov(ov_model, request);
784  if (ret != DNN_SUCCESS) {
785  av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
786  return ret;
787  }
788  status = ie_infer_set_completion_callback(request->infer_request, &request->callback);
789  if (status != OK) {
790  av_log(ctx, AV_LOG_ERROR, "Failed to set completion callback for inference\n");
791  return DNN_ERROR;
792  }
793  status = ie_infer_request_infer_async(request->infer_request);
794  if (status != OK) {
795  av_log(ctx, AV_LOG_ERROR, "Failed to start async inference\n");
796  return DNN_ERROR;
797  }
798 
799  return DNN_SUCCESS;
800 }
801 
803 {
804  if (*model){
805  OVModel *ov_model = (*model)->model;
806  while (ff_safe_queue_size(ov_model->request_queue) != 0) {
808  if (item && item->infer_request) {
809  ie_infer_request_free(&item->infer_request);
810  }
811  av_freep(&item->tasks);
812  av_freep(&item);
813  }
815 
816  while (ff_queue_size(ov_model->task_queue) != 0) {
817  TaskItem *item = ff_queue_pop_front(ov_model->task_queue);
818  av_frame_free(&item->in_frame);
819  av_frame_free(&item->out_frame);
820  av_freep(&item);
821  }
822  ff_queue_destroy(ov_model->task_queue);
823 
824  if (ov_model->infer_request)
825  ie_infer_request_free(&ov_model->infer_request);
826  if (ov_model->exe_network)
827  ie_exec_network_free(&ov_model->exe_network);
828  if (ov_model->network)
829  ie_network_free(&ov_model->network);
830  if (ov_model->core)
831  ie_core_free(&ov_model->core);
832  av_freep(&ov_model);
833  av_freep(model);
834  }
835 }
void * model
Definition: dnn_interface.h:71
#define NULL
Definition: coverity.c:32
DNNReturnType ff_dnn_execute_model_async_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
const char * output_name
Buffered I/O operations.
This structure describes decoded (raw) audio or video data.
Definition: frame.h:318
AVOption.
Definition: opt.h:248
int av_cpu_count(void)
Definition: cpu.c:275
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
Definition: opt.c:1358
int channels
Definition: dnn_interface.h:60
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
static int get_datatype_size(DNNDataType dt)
void ff_queue_destroy(Queue *q)
Definition: queue.c:73
int ff_queue_push_back(Queue *q, void *v)
Definition: queue.c:131
void * ff_queue_peek_front(Queue *q)
Definition: queue.c:94
SafeQueue * ff_safe_queue_create(void)
Definition: safe_queue.c:52
const char * options
Definition: dnn_interface.h:73
ie_complete_call_back_t callback
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1559
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
void ff_safe_queue_destroy(SafeQueue *sq)
Definition: safe_queue.c:69
uint8_t
#define av_malloc(s)
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:191
AVOptions.
static const AVOption dnn_openvino_options[]
Queue * task_queue
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
ie_infer_request_t * infer_request
size_t ff_safe_queue_size(SafeQueue *sq)
Definition: safe_queue.c:80
static DNNReturnType get_input_ov(void *model, DNNData *input, const char *input_name)
DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:26
DNNModel * model
ie_infer_request_t * infer_request
static DNNReturnType execute_model_ov(RequestItem *request)
static DNNReturnType get_output_ov(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
#define av_log(a,...)
DNNDataType
Definition: dnn_interface.h:37
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:80
int width
Definition: frame.h:376
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
DNNColorOrder order
Definition: dnn_interface.h:63
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Definition: safe_queue.c:95
const char * input_name
int height
Definition: dnn_interface.h:60
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:86
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:204
void * data
Definition: dnn_interface.h:59
simple assert() macros that are a bit more flexible than ISO C assert().
static FilteringContext * filter_ctx
Definition: transcoding.c:48
ie_core_t * core
Definition: queue.c:34
#define FLAGS
OVOptions options
static DNNDataType precision_to_datatype(precision_e precision)
DetectPostProc detect_post_proc
Definition: dnn_interface.h:91
DNNReturnType ff_dnn_execute_model_ov(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame)
DNNAsyncStatusType
Definition: dnn_interface.h:44
const AVClass * class
size_t ff_queue_size(Queue *q)
Definition: queue.c:89
OVContext ctx
AVFormatContext * ctx
Definition: movenc.c:48
#define OFFSET(x)
AVFrame * in_frame
void * ff_queue_pop_front(Queue *q)
Definition: queue.c:152
DNNReturnType
Definition: dnn_interface.h:33
AVFILTER_DEFINE_CLASS(dnn_openvino)
static DNNReturnType fill_model_input_ov(OVModel *ov_model, RequestItem *request)
ie_network_t * network
void * ff_safe_queue_pop_front(SafeQueue *sq)
Definition: safe_queue.c:105
AVFrame * out_frame
DNN input&output process between AVFrame and DNNData.
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(const uint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(const int16_t *) pi >> 8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(const int16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(const int32_t *) pi >> 24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(const int32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(const float *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(const float *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(const float *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(const double *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(const double *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(const double *) pi *(1U<< 31))))#define SET_CONV_FUNC_GROUP(ofmt, ifmt) static void set_generic_function(AudioConvert *ac){}void ff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enum AVSampleFormat out_fmt, enum AVSampleFormat in_fmt, int channels, int sample_rate, int apply_map){AudioConvert *ac;int in_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) return NULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt) > 2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);return NULL;}return ac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}else if(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;else ac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);return ac;}int ff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){int use_generic=1;int len=in->nb_samples;int p;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%d samples - audio_convert: %s to %s (dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));return ff_convert_dither(ac-> in
Describe the class of an AVClass context structure.
Definition: log.h:67
DNNFunctionType func_type
Definition: dnn_interface.h:77
DNNFunctionType
Definition: dnn_interface.h:51
DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, DNNFunctionType func_type, void *log_ctx)
Definition: dnn_io_proc.c:207
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
void ff_dnn_free_model_ov(DNNModel **model)
Queue * ff_queue_create(void)
Definition: queue.c:48
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
static DNNReturnType init_model_ov(OVModel *ov_model, const char *input_name, const char *output_name)
int ff_safe_queue_push_front(SafeQueue *sq, void *v)
Definition: safe_queue.c:85
const OptionDef options[]
Definition: ffmpeg_opt.c:3427
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:82
DNN inference functions interface for OpenVINO backend.
SafeQueue * request_queue
static void infer_completion_callback(void *args)
TaskItem ** tasks
#define APPEND_STRING(generated_string, iterate_string)
An instance of a filter.
Definition: avfilter.h:341
OVModel * ov_model
int height
Definition: frame.h:376
FILE * out
Definition: movenc.c:54
#define av_freep(p)
#define av_malloc_array(a, b)
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:89
DNNAsyncStatusType ff_dnn_get_async_result_ov(const DNNModel *model, AVFrame **in, AVFrame **out)
DNNModel * ff_dnn_load_model_ov(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
AVFilterContext * filter_ctx
Definition: dnn_interface.h:75
DNNReturnType ff_dnn_flush_ov(const DNNModel *model)
DNNDataType dt
Definition: dnn_interface.h:62
ie_executable_network_t * exe_network
int i
Definition: input.c:407