FFmpeg
dnn_backend_tf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN tensorflow backend implementation.
24  */
25 
26 #include "dnn_backend_tf.h"
27 #include "dnn_backend_native.h"
30 #include "libavformat/avio.h"
31 #include "libavutil/avassert.h"
32 #include "libavutil/avstring.h"
33 #include "libavutil/cpu.h"
34 #include "libavcodec/defs.h"
35 #include "../internal.h"
38 #include "dnn_io_proc.h"
39 #include "dnn_backend_common.h"
40 #include "safe_queue.h"
41 #include <tensorflow/c/c_api.h>
42 
43 typedef struct TFOptions{
44  char *sess_config;
45  uint8_t async;
46  uint32_t nireq;
47 } TFOptions;
48 
49 typedef struct TFContext {
50  const AVClass *class;
52 } TFContext;
53 
54 typedef struct TFModel{
57  TF_Graph *graph;
58  TF_Session *session;
59  TF_Status *status;
63 } TFModel;
64 
65 /**
66  * Stores execution parameters for single
67  * call to the TensorFlow C API
68  */
69 typedef struct TFInferRequest {
70  TF_Output *tf_outputs;
71  TF_Tensor **output_tensors;
72  TF_Output *tf_input;
73  TF_Tensor *input_tensor;
75 
76 typedef struct TFRequestItem {
79  TF_Status *status;
82 
83 #define OFFSET(x) offsetof(TFContext, x)
84 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
85 static const AVOption dnn_tensorflow_options[] = {
86  { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
88  { NULL }
89 };
90 
91 AVFILTER_DEFINE_CLASS(dnn_tensorflow);
92 
93 static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue);
94 static void infer_completion_callback(void *args);
95 static inline void destroy_request_item(TFRequestItem **arg);
96 
97 static void free_buffer(void *data, size_t length)
98 {
99  av_freep(&data);
100 }
101 
102 /**
103  * Free the contents of TensorFlow inference request.
104  * It does not free the TFInferRequest instance.
105  *
106  * @param request pointer to TFInferRequest instance.
107  * NULL pointer is allowed.
108  */
109 static void tf_free_request(TFInferRequest *request)
110 {
111  if (!request)
112  return;
113  if (request->input_tensor) {
114  TF_DeleteTensor(request->input_tensor);
115  request->input_tensor = NULL;
116  }
117  av_freep(&request->tf_input);
118  av_freep(&request->tf_outputs);
119  if (request->output_tensors) {
120  int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]);
121  for (uint32_t i = 0; i < nb_output; ++i) {
122  if (request->output_tensors[i]) {
123  TF_DeleteTensor(request->output_tensors[i]);
124  request->output_tensors[i] = NULL;
125  }
126  }
127  av_freep(&request->output_tensors);
128  }
129 }
130 
131 /**
132  * Create a TensorFlow inference request. All properties
133  * are initially unallocated and set as NULL.
134  *
135  * @return pointer to the allocated TFInferRequest instance.
136  */
138 {
139  TFInferRequest *infer_request = av_malloc(sizeof(TFInferRequest));
140  if (!infer_request) {
141  return NULL;
142  }
143  infer_request->tf_outputs = NULL;
144  infer_request->tf_input = NULL;
145  infer_request->input_tensor = NULL;
146  infer_request->output_tensors = NULL;
147  return infer_request;
148 }
149 
150 /**
151  * Start synchronous inference for the TensorFlow model.
152  *
153  * @param request pointer to the TFRequestItem for inference
154  * @retval 0 if execution is successful
155  * @retval AVERROR(EINVAL) if request is NULL
156  * @retval DNN_GENERIC_ERROR if execution fails
157  */
158 static int tf_start_inference(void *args)
159 {
160  TFRequestItem *request = args;
161  TFInferRequest *infer_request = request->infer_request;
162  LastLevelTaskItem *lltask = request->lltask;
163  TaskItem *task = lltask->task;
164  TFModel *tf_model = task->model;
165 
166  if (!request) {
167  av_log(&tf_model->ctx, AV_LOG_ERROR, "TFRequestItem is NULL\n");
168  return AVERROR(EINVAL);
169  }
170 
171  TF_SessionRun(tf_model->session, NULL,
172  infer_request->tf_input, &infer_request->input_tensor, 1,
173  infer_request->tf_outputs, infer_request->output_tensors,
174  task->nb_output, NULL, 0, NULL,
175  request->status);
176  if (TF_GetCode(request->status) != TF_OK) {
177  av_log(&tf_model->ctx, AV_LOG_ERROR, "%s", TF_Message(request->status));
178  tf_free_request(infer_request);
179  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
180  destroy_request_item(&request);
181  }
182  return DNN_GENERIC_ERROR;
183  }
184  return 0;
185 }
186 
187 /**
188  * Free the TFRequestItem completely.
189  *
190  * @param arg Address of the TFInferRequest instance.
191  */
192 static inline void destroy_request_item(TFRequestItem **arg) {
193  TFRequestItem *request;
194  if (!arg) {
195  return;
196  }
197  request = *arg;
198  tf_free_request(request->infer_request);
199  av_freep(&request->infer_request);
200  av_freep(&request->lltask);
201  TF_DeleteStatus(request->status);
203  av_freep(arg);
204 }
205 
206 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
207 {
208  TFModel *tf_model = task->model;
209  TFContext *ctx = &tf_model->ctx;
210  LastLevelTaskItem *lltask = av_malloc(sizeof(*lltask));
211  if (!lltask) {
212  av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for LastLevelTaskItem\n");
213  return AVERROR(ENOMEM);
214  }
215  task->inference_todo = 1;
216  task->inference_done = 0;
217  lltask->task = task;
218  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
219  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
220  av_freep(&lltask);
221  return AVERROR(ENOMEM);
222  }
223  return 0;
224 }
225 
226 static TF_Buffer *read_graph(const char *model_filename)
227 {
228  TF_Buffer *graph_buf;
229  unsigned char *graph_data = NULL;
230  AVIOContext *model_file_context;
231  long size, bytes_read;
232 
233  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
234  return NULL;
235  }
236 
237  size = avio_size(model_file_context);
238 
239  graph_data = av_malloc(size);
240  if (!graph_data){
241  avio_closep(&model_file_context);
242  return NULL;
243  }
244  bytes_read = avio_read(model_file_context, graph_data, size);
245  avio_closep(&model_file_context);
246  if (bytes_read != size){
247  av_freep(&graph_data);
248  return NULL;
249  }
250 
251  graph_buf = TF_NewBuffer();
252  graph_buf->data = graph_data;
253  graph_buf->length = size;
254  graph_buf->data_deallocator = free_buffer;
255 
256  return graph_buf;
257 }
258 
259 static TF_Tensor *allocate_input_tensor(const DNNData *input)
260 {
261  TF_DataType dt;
262  size_t size;
263  int64_t input_dims[] = {1, input->height, input->width, input->channels};
264  switch (input->dt) {
265  case DNN_FLOAT:
266  dt = TF_FLOAT;
267  size = sizeof(float);
268  break;
269  case DNN_UINT8:
270  dt = TF_UINT8;
271  size = 1;
272  break;
273  default:
274  av_assert0(!"should not reach here");
275  }
276 
277  return TF_AllocateTensor(dt, input_dims, 4,
278  input_dims[1] * input_dims[2] * input_dims[3] * size);
279 }
280 
281 static int get_input_tf(void *model, DNNData *input, const char *input_name)
282 {
283  TFModel *tf_model = model;
284  TFContext *ctx = &tf_model->ctx;
285  TF_Status *status;
286  int64_t dims[4];
287 
288  TF_Output tf_output;
289  tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);
290  if (!tf_output.oper) {
291  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
292  return AVERROR(EINVAL);
293  }
294 
295  tf_output.index = 0;
296  input->dt = TF_OperationOutputType(tf_output);
297  input->order = DCO_RGB;
298 
299  status = TF_NewStatus();
300  TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);
301  if (TF_GetCode(status) != TF_OK){
302  TF_DeleteStatus(status);
303  av_log(ctx, AV_LOG_ERROR, "Failed to get input tensor shape: number of dimension incorrect\n");
304  return DNN_GENERIC_ERROR;
305  }
306  TF_DeleteStatus(status);
307 
308  // currently only NHWC is supported
309  av_assert0(dims[0] == 1 || dims[0] == -1);
310  input->height = dims[1];
311  input->width = dims[2];
312  input->channels = dims[3];
313 
314  return 0;
315 }
316 
317 static int get_output_tf(void *model, const char *input_name, int input_width, int input_height,
318  const char *output_name, int *output_width, int *output_height)
319 {
320  int ret;
321  TFModel *tf_model = model;
322  TFContext *ctx = &tf_model->ctx;
323  TaskItem task;
324  TFRequestItem *request;
325  DNNExecBaseParams exec_params = {
326  .input_name = input_name,
327  .output_names = &output_name,
328  .nb_output = 1,
329  .in_frame = NULL,
330  .out_frame = NULL,
331  };
332 
333  ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, tf_model, input_height, input_width, ctx);
334  if (ret != 0) {
335  goto err;
336  }
337 
338  ret = extract_lltask_from_task(&task, tf_model->lltask_queue);
339  if (ret != 0) {
340  av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
341  goto err;
342  }
343 
344  request = ff_safe_queue_pop_front(tf_model->request_queue);
345  if (!request) {
346  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
347  ret = AVERROR(EINVAL);
348  goto err;
349  }
350 
351  ret = execute_model_tf(request, tf_model->lltask_queue);
352  *output_width = task.out_frame->width;
353  *output_height = task.out_frame->height;
354 
355 err:
356  av_frame_free(&task.out_frame);
357  av_frame_free(&task.in_frame);
358  return ret;
359 }
360 
361 #define SPACE_CHARS " \t\r\n"
362 static int hex_to_data(uint8_t *data, const char *p)
363 {
364  int c, len, v;
365 
366  len = 0;
367  v = 1;
368  for (;;) {
369  p += strspn(p, SPACE_CHARS);
370  if (*p == '\0')
371  break;
372  c = av_toupper((unsigned char) *p++);
373  if (c >= '0' && c <= '9')
374  c = c - '0';
375  else if (c >= 'A' && c <= 'F')
376  c = c - 'A' + 10;
377  else
378  break;
379  v = (v << 4) | c;
380  if (v & 0x100) {
381  if (data) {
382  data[len] = v;
383  }
384  len++;
385  v = 1;
386  }
387  }
388  return len;
389 }
390 
391 static int load_tf_model(TFModel *tf_model, const char *model_filename)
392 {
393  TFContext *ctx = &tf_model->ctx;
394  TF_Buffer *graph_def;
395  TF_ImportGraphDefOptions *graph_opts;
396  TF_SessionOptions *sess_opts;
397  const TF_Operation *init_op;
398  uint8_t *sess_config = NULL;
399  int sess_config_length = 0;
400 
401  // prepare the sess config data
402  if (tf_model->ctx.options.sess_config != NULL) {
403  const char *config;
404  /*
405  tf_model->ctx.options.sess_config is hex to present the serialized proto
406  required by TF_SetConfig below, so we need to first generate the serialized
407  proto in a python script, tools/python/tf_sess_config.py is a script example
408  to generate the configs of sess_config.
409  */
410  if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {
411  av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");
412  return AVERROR(EINVAL);
413  }
414  config = tf_model->ctx.options.sess_config + 2;
415  sess_config_length = hex_to_data(NULL, config);
416 
417  sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);
418  if (!sess_config) {
419  av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");
420  return AVERROR(ENOMEM);
421  }
422  if (hex_to_data(sess_config, config) < 0) {
423  av_log(ctx, AV_LOG_ERROR, "failed to convert hex to data\n");
424  return AVERROR(EINVAL);
425  }
426  }
427 
428  graph_def = read_graph(model_filename);
429  if (!graph_def){
430  av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);
431  av_freep(&sess_config);
432  return AVERROR(EINVAL);
433  }
434  tf_model->graph = TF_NewGraph();
435  tf_model->status = TF_NewStatus();
436  graph_opts = TF_NewImportGraphDefOptions();
437  TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
438  TF_DeleteImportGraphDefOptions(graph_opts);
439  TF_DeleteBuffer(graph_def);
440  if (TF_GetCode(tf_model->status) != TF_OK){
441  TF_DeleteGraph(tf_model->graph);
442  TF_DeleteStatus(tf_model->status);
443  av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");
444  av_freep(&sess_config);
445  return DNN_GENERIC_ERROR;
446  }
447 
448  init_op = TF_GraphOperationByName(tf_model->graph, "init");
449  sess_opts = TF_NewSessionOptions();
450 
451  if (sess_config) {
452  TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);
453  av_freep(&sess_config);
454  if (TF_GetCode(tf_model->status) != TF_OK) {
455  TF_DeleteGraph(tf_model->graph);
456  TF_DeleteStatus(tf_model->status);
457  TF_DeleteSessionOptions(sess_opts);
458  av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",
459  tf_model->ctx.options.sess_config);
460  return DNN_GENERIC_ERROR;
461  }
462  }
463 
464  tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
465  TF_DeleteSessionOptions(sess_opts);
466  if (TF_GetCode(tf_model->status) != TF_OK)
467  {
468  TF_DeleteGraph(tf_model->graph);
469  TF_DeleteStatus(tf_model->status);
470  av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");
471  return DNN_GENERIC_ERROR;
472  }
473 
474  // Run initialization operation with name "init" if it is present in graph
475  if (init_op){
476  TF_SessionRun(tf_model->session, NULL,
477  NULL, NULL, 0,
478  NULL, NULL, 0,
479  &init_op, 1, NULL, tf_model->status);
480  if (TF_GetCode(tf_model->status) != TF_OK)
481  {
482  TF_DeleteSession(tf_model->session, tf_model->status);
483  TF_DeleteGraph(tf_model->graph);
484  TF_DeleteStatus(tf_model->status);
485  av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");
486  return DNN_GENERIC_ERROR;
487  }
488  }
489 
490  return 0;
491 }
492 
493 #define NAME_BUFFER_SIZE 256
494 
495 static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
496  ConvolutionalParams* params, const int layer)
497 {
498  TFContext *ctx = &tf_model->ctx;
499  TF_Operation *op;
500  TF_OperationDescription *op_desc;
501  TF_Output input;
502  int64_t strides[] = {1, 1, 1, 1};
503  TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
504  int64_t dims[4];
505  int dims_len;
506  char name_buffer[NAME_BUFFER_SIZE];
507  int32_t size;
508 
509  size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
510  input.index = 0;
511 
512  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
513  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
514  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
515  dims[0] = params->output_num;
516  dims[1] = params->kernel_size;
517  dims[2] = params->kernel_size;
518  dims[3] = params->input_num;
519  dims_len = 4;
520  kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
521  memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
522  TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
523  if (TF_GetCode(tf_model->status) != TF_OK){
524  goto err;
525  }
526  op = TF_FinishOperation(op_desc, tf_model->status);
527  if (TF_GetCode(tf_model->status) != TF_OK){
528  goto err;
529  }
530 
531  snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
532  op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
533  input.oper = op;
534  TF_AddInput(op_desc, input);
535  input.oper = transpose_op;
536  TF_AddInput(op_desc, input);
537  TF_SetAttrType(op_desc, "T", TF_FLOAT);
538  TF_SetAttrType(op_desc, "Tperm", TF_INT32);
539  op = TF_FinishOperation(op_desc, tf_model->status);
540  if (TF_GetCode(tf_model->status) != TF_OK){
541  goto err;
542  }
543 
544  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
545  op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
546  input.oper = *cur_op;
547  TF_AddInput(op_desc, input);
548  input.oper = op;
549  TF_AddInput(op_desc, input);
550  TF_SetAttrType(op_desc, "T", TF_FLOAT);
551  TF_SetAttrIntList(op_desc, "strides", strides, 4);
552  TF_SetAttrString(op_desc, "padding", "VALID", 5);
553  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
554  if (TF_GetCode(tf_model->status) != TF_OK){
555  goto err;
556  }
557 
558  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
559  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
560  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
561  dims[0] = params->output_num;
562  dims_len = 1;
563  biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
564  memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
565  TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
566  if (TF_GetCode(tf_model->status) != TF_OK){
567  goto err;
568  }
569  op = TF_FinishOperation(op_desc, tf_model->status);
570  if (TF_GetCode(tf_model->status) != TF_OK){
571  goto err;
572  }
573 
574  snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
575  op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
576  input.oper = *cur_op;
577  TF_AddInput(op_desc, input);
578  input.oper = op;
579  TF_AddInput(op_desc, input);
580  TF_SetAttrType(op_desc, "T", TF_FLOAT);
581  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
582  if (TF_GetCode(tf_model->status) != TF_OK){
583  goto err;
584  }
585 
586  snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
587  switch (params->activation){
588  case RELU:
589  op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
590  break;
591  case TANH:
592  op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
593  break;
594  case SIGMOID:
595  op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
596  break;
597  default:
598  avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
599  return AVERROR(ENOSYS);
600  }
601  input.oper = *cur_op;
602  TF_AddInput(op_desc, input);
603  TF_SetAttrType(op_desc, "T", TF_FLOAT);
604  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
605  if (TF_GetCode(tf_model->status) != TF_OK){
606  goto err;
607  }
608 
609  return 0;
610 err:
611  TF_DeleteTensor(kernel_tensor);
612  TF_DeleteTensor(biases_tensor);
613  av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
614  return DNN_GENERIC_ERROR;
615 }
616 
617 static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
618  DepthToSpaceParams *params, const int layer)
619 {
620  TFContext *ctx = &tf_model->ctx;
621  TF_OperationDescription *op_desc;
622  TF_Output input;
623  char name_buffer[NAME_BUFFER_SIZE];
624 
625  snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
626  op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
627  input.oper = *cur_op;
628  input.index = 0;
629  TF_AddInput(op_desc, input);
630  TF_SetAttrType(op_desc, "T", TF_FLOAT);
631  TF_SetAttrInt(op_desc, "block_size", params->block_size);
632  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
633  if (TF_GetCode(tf_model->status) != TF_OK){
634  av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);
635  return DNN_GENERIC_ERROR;
636  }
637 
638  return 0;
639 }
640 
641 static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
642  LayerPadParams *params, const int layer)
643 {
644  TFContext *ctx = &tf_model->ctx;
645  TF_Operation *op;
646  TF_Tensor *tensor;
647  TF_OperationDescription *op_desc;
648  TF_Output input;
649  int32_t *pads;
650  int64_t pads_shape[] = {4, 2};
651 
652  char name_buffer[NAME_BUFFER_SIZE];
653  snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
654 
655  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
656  TF_SetAttrType(op_desc, "dtype", TF_INT32);
657  tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
658  pads = (int32_t *)TF_TensorData(tensor);
659  pads[0] = params->paddings[0][0];
660  pads[1] = params->paddings[0][1];
661  pads[2] = params->paddings[1][0];
662  pads[3] = params->paddings[1][1];
663  pads[4] = params->paddings[2][0];
664  pads[5] = params->paddings[2][1];
665  pads[6] = params->paddings[3][0];
666  pads[7] = params->paddings[3][1];
667  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
668  if (TF_GetCode(tf_model->status) != TF_OK){
669  TF_DeleteTensor(tensor);
670  av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
671  return DNN_GENERIC_ERROR;
672  }
673  op = TF_FinishOperation(op_desc, tf_model->status);
674  if (TF_GetCode(tf_model->status) != TF_OK){
675  TF_DeleteTensor(tensor);
676  av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
677  return DNN_GENERIC_ERROR;
678  }
679 
680  op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
681  input.oper = *cur_op;
682  input.index = 0;
683  TF_AddInput(op_desc, input);
684  input.oper = op;
685  TF_AddInput(op_desc, input);
686  TF_SetAttrType(op_desc, "T", TF_FLOAT);
687  TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
688  TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
689  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
690  if (TF_GetCode(tf_model->status) != TF_OK){
691  TF_DeleteTensor(tensor);
692  av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
693  return DNN_GENERIC_ERROR;
694  }
695 
696  return 0;
697 }
698 
699 static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
700  DnnLayerMaximumParams *params, const int layer)
701 {
702  TFContext *ctx = &tf_model->ctx;
703  TF_Operation *op;
704  TF_Tensor *tensor;
705  TF_OperationDescription *op_desc;
706  TF_Output input;
707  float *y;
708 
709  char name_buffer[NAME_BUFFER_SIZE];
710  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
711 
712  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
713  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
714  tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
715  y = (float *)TF_TensorData(tensor);
716  *y = params->val.y;
717  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
718  if (TF_GetCode(tf_model->status) != TF_OK){
719  TF_DeleteTensor(tensor);
720  av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
721  return DNN_GENERIC_ERROR;
722  }
723  op = TF_FinishOperation(op_desc, tf_model->status);
724  if (TF_GetCode(tf_model->status) != TF_OK){
725  TF_DeleteTensor(tensor);
726  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
727  return DNN_GENERIC_ERROR;
728  }
729 
730  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
731  op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
732  input.oper = *cur_op;
733  input.index = 0;
734  TF_AddInput(op_desc, input);
735  input.oper = op;
736  TF_AddInput(op_desc, input);
737  TF_SetAttrType(op_desc, "T", TF_FLOAT);
738  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
739  if (TF_GetCode(tf_model->status) != TF_OK){
740  TF_DeleteTensor(tensor);
741  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
742  return DNN_GENERIC_ERROR;
743  }
744 
745  return 0;
746 }
747 
748 static int load_native_model(TFModel *tf_model, const char *model_filename)
749 {
750  TFContext *ctx = &tf_model->ctx;
751  int32_t layer;
752  TF_OperationDescription *op_desc;
753  TF_Operation *op;
754  TF_Operation *transpose_op;
755  TF_Tensor *tensor = NULL;
756  TF_Output input;
758  int64_t transpose_perm_shape[] = {4};
759  int64_t input_shape[] = {1, -1, -1, -1};
760  int layer_add_res;
761  DNNModel *model = NULL;
762  NativeModel *native_model;
763 
764  model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
765  if (!model){
766  av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
767  return AVERROR(EINVAL);
768  }
769 
770  native_model = model->model;
771  tf_model->graph = TF_NewGraph();
772  tf_model->status = TF_NewStatus();
773 
774 #define CLEANUP_ON_ERROR(tf_model) \
775  { \
776  TF_DeleteTensor(tensor); \
777  TF_DeleteGraph(tf_model->graph); \
778  TF_DeleteStatus(tf_model->status); \
779  av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
780  return DNN_GENERIC_ERROR; \
781  }
782 
783  op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
784  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
785  TF_SetAttrShape(op_desc, "shape", input_shape, 4);
786  op = TF_FinishOperation(op_desc, tf_model->status);
787  if (TF_GetCode(tf_model->status) != TF_OK){
788  CLEANUP_ON_ERROR(tf_model);
789  }
790 
791  op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
792  TF_SetAttrType(op_desc, "dtype", TF_INT32);
793  tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
794  transpose_perm = (int32_t *)TF_TensorData(tensor);
795  transpose_perm[0] = 1;
796  transpose_perm[1] = 2;
797  transpose_perm[2] = 3;
798  transpose_perm[3] = 0;
799  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
800  if (TF_GetCode(tf_model->status) != TF_OK){
801  CLEANUP_ON_ERROR(tf_model);
802  }
803  transpose_op = TF_FinishOperation(op_desc, tf_model->status);
804  if (TF_GetCode(tf_model->status) != TF_OK){
805  CLEANUP_ON_ERROR(tf_model);
806  }
807 
808  for (layer = 0; layer < native_model->layers_num; ++layer){
809  switch (native_model->layers[layer].type){
810  case DLT_INPUT:
811  layer_add_res = 0;
812  break;
813  case DLT_CONV2D:
814  layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
815  (ConvolutionalParams *)native_model->layers[layer].params, layer);
816  break;
817  case DLT_DEPTH_TO_SPACE:
818  layer_add_res = add_depth_to_space_layer(tf_model, &op,
819  (DepthToSpaceParams *)native_model->layers[layer].params, layer);
820  break;
821  case DLT_MIRROR_PAD:
822  layer_add_res = add_pad_layer(tf_model, &op,
823  (LayerPadParams *)native_model->layers[layer].params, layer);
824  break;
825  case DLT_MAXIMUM:
826  layer_add_res = add_maximum_layer(tf_model, &op,
827  (DnnLayerMaximumParams *)native_model->layers[layer].params, layer);
828  break;
829  default:
830  CLEANUP_ON_ERROR(tf_model);
831  }
832 
833  if (layer_add_res != 0){
834  CLEANUP_ON_ERROR(tf_model);
835  }
836  }
837 
838  op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
839  input.oper = op;
840  input.index = 0;
841  TF_AddInput(op_desc, input);
842  TF_FinishOperation(op_desc, tf_model->status);
843  if (TF_GetCode(tf_model->status) != TF_OK){
844  CLEANUP_ON_ERROR(tf_model);
845  }
846 
847  ff_dnn_free_model_native(&model);
848 
849  return 0;
850 }
851 
852 DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
853 {
854  DNNModel *model = NULL;
855  TFModel *tf_model = NULL;
856  TFContext *ctx = NULL;
857 
858  model = av_mallocz(sizeof(DNNModel));
859  if (!model){
860  return NULL;
861  }
862 
863  tf_model = av_mallocz(sizeof(TFModel));
864  if (!tf_model){
865  av_freep(&model);
866  return NULL;
867  }
868  tf_model->model = model;
869  ctx = &tf_model->ctx;
870  ctx->class = &dnn_tensorflow_class;
871 
872  //parse options
874  if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {
875  av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
876  goto err;
877  }
878 
879  if (load_tf_model(tf_model, model_filename) != 0){
880  if (load_native_model(tf_model, model_filename) != 0){
881  goto err;
882  }
883  }
884 
885  if (ctx->options.nireq <= 0) {
886  ctx->options.nireq = av_cpu_count() / 2 + 1;
887  }
888 
889 #if !HAVE_PTHREAD_CANCEL
890  if (ctx->options.async) {
891  ctx->options.async = 0;
892  av_log(filter_ctx, AV_LOG_WARNING, "pthread is not supported, roll back to sync.\n");
893  }
894 #endif
895 
896  tf_model->request_queue = ff_safe_queue_create();
897  if (!tf_model->request_queue) {
898  goto err;
899  }
900 
901  for (int i = 0; i < ctx->options.nireq; i++) {
902  TFRequestItem *item = av_mallocz(sizeof(*item));
903  if (!item) {
904  goto err;
905  }
906  item->lltask = NULL;
908  if (!item->infer_request) {
909  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for TensorFlow inference request\n");
910  av_freep(&item);
911  goto err;
912  }
913  item->status = TF_NewStatus();
916  item->exec_module.args = item;
917 
918  if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {
919  destroy_request_item(&item);
920  goto err;
921  }
922  }
923 
924  tf_model->lltask_queue = ff_queue_create();
925  if (!tf_model->lltask_queue) {
926  goto err;
927  }
928 
929  tf_model->task_queue = ff_queue_create();
930  if (!tf_model->task_queue) {
931  goto err;
932  }
933 
934  model->model = tf_model;
935  model->get_input = &get_input_tf;
936  model->get_output = &get_output_tf;
937  model->options = options;
938  model->filter_ctx = filter_ctx;
939  model->func_type = func_type;
940 
941  return model;
942 err:
943  ff_dnn_free_model_tf(&model);
944  return NULL;
945 }
946 
947 static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request) {
948  DNNData input;
949  LastLevelTaskItem *lltask;
950  TaskItem *task;
951  TFInferRequest *infer_request;
952  TFContext *ctx = &tf_model->ctx;
953  int ret = 0;
954 
955  lltask = ff_queue_pop_front(tf_model->lltask_queue);
956  av_assert0(lltask);
957  task = lltask->task;
958  request->lltask = lltask;
959 
960  ret = get_input_tf(tf_model, &input, task->input_name);
961  if (ret != 0) {
962  goto err;
963  }
964 
965  infer_request = request->infer_request;
966  input.height = task->in_frame->height;
967  input.width = task->in_frame->width;
968 
969  infer_request->tf_input = av_malloc(sizeof(TF_Output));
970  if (!infer_request->tf_input) {
971  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
972  ret = AVERROR(ENOMEM);
973  goto err;
974  }
975 
976  infer_request->tf_input->oper = TF_GraphOperationByName(tf_model->graph, task->input_name);
977  if (!infer_request->tf_input->oper){
978  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
980  goto err;
981  }
982  infer_request->tf_input->index = 0;
983 
984  infer_request->input_tensor = allocate_input_tensor(&input);
985  if (!infer_request->input_tensor){
986  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
987  ret = AVERROR(ENOMEM);
988  goto err;
989  }
990  input.data = (float *)TF_TensorData(infer_request->input_tensor);
991 
992  switch (tf_model->model->func_type) {
993  case DFT_PROCESS_FRAME:
994  if (task->do_ioproc) {
995  if (tf_model->model->frame_pre_proc != NULL) {
996  tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx);
997  } else {
999  }
1000  }
1001  break;
1002  case DFT_ANALYTICS_DETECT:
1004  break;
1005  default:
1006  avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type);
1007  break;
1008  }
1009 
1010  infer_request->tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output));
1011  if (infer_request->tf_outputs == NULL) {
1012  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n");
1013  ret = AVERROR(ENOMEM);
1014  goto err;
1015  }
1016 
1017  infer_request->output_tensors = av_calloc(task->nb_output, sizeof(*infer_request->output_tensors));
1018  if (!infer_request->output_tensors) {
1019  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n");
1020  ret = AVERROR(ENOMEM);
1021  goto err;
1022  }
1023 
1024  for (int i = 0; i < task->nb_output; ++i) {
1025  infer_request->output_tensors[i] = NULL;
1026  infer_request->tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]);
1027  if (!infer_request->tf_outputs[i].oper) {
1028  av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]);
1030  goto err;
1031  }
1032  infer_request->tf_outputs[i].index = 0;
1033  }
1034 
1035  return 0;
1036 err:
1037  tf_free_request(infer_request);
1038  return ret;
1039 }
1040 
1041 static void infer_completion_callback(void *args) {
1042  TFRequestItem *request = args;
1043  LastLevelTaskItem *lltask = request->lltask;
1044  TaskItem *task = lltask->task;
1045  DNNData *outputs;
1046  TFInferRequest *infer_request = request->infer_request;
1047  TFModel *tf_model = task->model;
1048  TFContext *ctx = &tf_model->ctx;
1049 
1050  outputs = av_malloc_array(task->nb_output, sizeof(*outputs));
1051  if (!outputs) {
1052  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *outputs\n");
1053  goto err;
1054  }
1055 
1056  for (uint32_t i = 0; i < task->nb_output; ++i) {
1057  outputs[i].height = TF_Dim(infer_request->output_tensors[i], 1);
1058  outputs[i].width = TF_Dim(infer_request->output_tensors[i], 2);
1059  outputs[i].channels = TF_Dim(infer_request->output_tensors[i], 3);
1060  outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
1061  outputs[i].dt = TF_TensorType(infer_request->output_tensors[i]);
1062  }
1063  switch (tf_model->model->func_type) {
1064  case DFT_PROCESS_FRAME:
1065  //it only support 1 output if it's frame in & frame out
1066  if (task->do_ioproc) {
1067  if (tf_model->model->frame_post_proc != NULL) {
1068  tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx);
1069  } else {
1071  }
1072  } else {
1073  task->out_frame->width = outputs[0].width;
1074  task->out_frame->height = outputs[0].height;
1075  }
1076  break;
1077  case DFT_ANALYTICS_DETECT:
1078  if (!tf_model->model->detect_post_proc) {
1079  av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
1080  return;
1081  }
1082  tf_model->model->detect_post_proc(task->in_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
1083  break;
1084  default:
1085  av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
1086  goto err;
1087  }
1088  task->inference_done++;
1089 err:
1090  tf_free_request(infer_request);
1091  av_freep(&outputs);
1092 
1093  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1094  destroy_request_item(&request);
1095  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
1096  }
1097 }
1098 
1099 static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
1100 {
1101  TFModel *tf_model;
1102  TFContext *ctx;
1103  LastLevelTaskItem *lltask;
1104  TaskItem *task;
1105  int ret = 0;
1106 
1107  if (ff_queue_size(lltask_queue) == 0) {
1108  destroy_request_item(&request);
1109  return 0;
1110  }
1111 
1112  lltask = ff_queue_peek_front(lltask_queue);
1113  task = lltask->task;
1114  tf_model = task->model;
1115  ctx = &tf_model->ctx;
1116 
1117  ret = fill_model_input_tf(tf_model, request);
1118  if (ret != 0) {
1119  goto err;
1120  }
1121 
1122  if (task->async) {
1123  if (ff_dnn_start_inference_async(ctx, &request->exec_module) != 0) {
1124  goto err;
1125  }
1126  return 0;
1127  }
1128  else {
1129  ret = tf_start_inference(request);
1130  if (ret != 0) {
1131  goto err;
1132  }
1133  infer_completion_callback(request);
1134  return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
1135  }
1136 err:
1137  tf_free_request(request->infer_request);
1138  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1139  destroy_request_item(&request);
1140  }
1141  return ret;
1142 }
1143 
1144 int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
1145 {
1146  TFModel *tf_model = model->model;
1147  TFContext *ctx = &tf_model->ctx;
1148  TaskItem *task;
1149  TFRequestItem *request;
1150  int ret = 0;
1151 
1152  ret = ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params);
1153  if (ret != 0) {
1154  return ret;
1155  }
1156 
1157  task = av_malloc(sizeof(*task));
1158  if (!task) {
1159  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
1160  return AVERROR(ENOMEM);
1161  }
1162 
1163  ret = ff_dnn_fill_task(task, exec_params, tf_model, ctx->options.async, 1);
1164  if (ret != 0) {
1165  av_freep(&task);
1166  return ret;
1167  }
1168 
1169  if (ff_queue_push_back(tf_model->task_queue, task) < 0) {
1170  av_freep(&task);
1171  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
1172  return AVERROR(ENOMEM);
1173  }
1174 
1175  ret = extract_lltask_from_task(task, tf_model->lltask_queue);
1176  if (ret != 0) {
1177  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
1178  return ret;
1179  }
1180 
1181  request = ff_safe_queue_pop_front(tf_model->request_queue);
1182  if (!request) {
1183  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1184  return AVERROR(EINVAL);
1185  }
1186  return execute_model_tf(request, tf_model->lltask_queue);
1187 }
1188 
1190 {
1191  TFModel *tf_model = model->model;
1192  return ff_dnn_get_result_common(tf_model->task_queue, in, out);
1193 }
1194 
1195 int ff_dnn_flush_tf(const DNNModel *model)
1196 {
1197  TFModel *tf_model = model->model;
1198  TFContext *ctx = &tf_model->ctx;
1199  TFRequestItem *request;
1200  int ret;
1201 
1202  if (ff_queue_size(tf_model->lltask_queue) == 0) {
1203  // no pending task need to flush
1204  return 0;
1205  }
1206 
1207  request = ff_safe_queue_pop_front(tf_model->request_queue);
1208  if (!request) {
1209  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1210  return AVERROR(EINVAL);
1211  }
1212 
1213  ret = fill_model_input_tf(tf_model, request);
1214  if (ret != 0) {
1215  av_log(ctx, AV_LOG_ERROR, "Failed to fill model input.\n");
1216  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1217  destroy_request_item(&request);
1218  }
1219  return ret;
1220  }
1221 
1222  return ff_dnn_start_inference_async(ctx, &request->exec_module);
1223 }
1224 
1226 {
1227  TFModel *tf_model;
1228 
1229  if (*model){
1230  tf_model = (*model)->model;
1231  while (ff_safe_queue_size(tf_model->request_queue) != 0) {
1233  destroy_request_item(&item);
1234  }
1236 
1237  while (ff_queue_size(tf_model->lltask_queue) != 0) {
1239  av_freep(&item);
1240  }
1241  ff_queue_destroy(tf_model->lltask_queue);
1242 
1243  while (ff_queue_size(tf_model->task_queue) != 0) {
1244  TaskItem *item = ff_queue_pop_front(tf_model->task_queue);
1245  av_frame_free(&item->in_frame);
1246  av_frame_free(&item->out_frame);
1247  av_freep(&item);
1248  }
1249  ff_queue_destroy(tf_model->task_queue);
1250 
1251  if (tf_model->graph){
1252  TF_DeleteGraph(tf_model->graph);
1253  }
1254  if (tf_model->session){
1255  TF_CloseSession(tf_model->session, tf_model->status);
1256  TF_DeleteSession(tf_model->session, tf_model->status);
1257  }
1258  if (tf_model->status){
1259  TF_DeleteStatus(tf_model->status);
1260  }
1261  av_freep(&tf_model);
1262  av_freep(model);
1263  }
1264 }
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(dnn_tensorflow)
ff_dnn_get_result_tf
DNNAsyncStatusType ff_dnn_get_result_tf(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_tf.c:1189
TFOptions::sess_config
char * sess_config
Definition: dnn_backend_tf.c:44
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:186
TFInferRequest
Stores execution parameters for single call to the TensorFlow C API.
Definition: dnn_backend_tf.c:69
TFInferRequest::tf_outputs
TF_Output * tf_outputs
Definition: dnn_backend_tf.c:70
execute_model_tf
static int execute_model_tf(TFRequestItem *request, Queue *lltask_queue)
Definition: dnn_backend_tf.c:1099
FLAGS
#define FLAGS
Definition: dnn_backend_tf.c:84
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
filter_ctx
static FilteringContext * filter_ctx
Definition: transcoding.c:49
TFModel::graph
TF_Graph * graph
Definition: dnn_backend_tf.c:57
av_opt_set_defaults
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
Definition: opt.c:1458
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
out
FILE * out
Definition: movenc.c:54
TFModel::ctx
TFContext ctx
Definition: dnn_backend_tf.c:55
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:58
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
extract_lltask_from_task
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_tf.c:206
add_conv_layer
static int add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
Definition: dnn_backend_tf.c:495
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:29
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DnnLayerMaximumParams::y
float y
Definition: dnn_backend_native_layer_maximum.h:36
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:111
LastLevelTaskItem
Definition: dnn_backend_common.h:50
ConvolutionalParams::kernel
float * kernel
Definition: dnn_backend_native_layer_conv2d.h:33
test::height
int height
Definition: vc1dsp.c:39
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:325
AVFrame::width
int width
Definition: frame.h:397
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:139
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
AVOption
AVOption.
Definition: opt.h:251
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
TFInferRequest::input_tensor
TF_Tensor * input_tensor
Definition: dnn_backend_tf.c:73
data
const char data[16]
Definition: mxf.c:143
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
load_tf_model
static int load_tf_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:391
dnn_io_proc.h
TFModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_tf.c:60
TaskItem
Definition: dnn_backend_common.h:36
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:70
DnnLayerMaximumParams
Definition: dnn_backend_native_layer_maximum.h:33
avio_size
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:352
tf_sess_config.config
config
Definition: tf_sess_config.py:33
OFFSET
#define OFFSET(x)
Definition: dnn_backend_tf.c:83
load_native_model
static int load_native_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:748
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
dnn_backend_native_layer_pad.h
destroy_request_item
static void destroy_request_item(TFRequestItem **arg)
Free the TFRequestItem completely.
Definition: dnn_backend_tf.c:192
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
DLT_INPUT
@ DLT_INPUT
Definition: dnn_backend_native.h:41
ff_dnn_execute_model_tf
int ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_tf.c:1144
TaskItem::model
void * model
Definition: dnn_backend_common.h:37
get_input_tf
static int get_input_tf(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_tf.c:281
SIGMOID
@ SIGMOID
Definition: dnn_backend_native.h:55
SPACE_CHARS
#define SPACE_CHARS
Definition: dnn_backend_tf.c:361
Queue
Linear double-ended data structure.
Definition: queue.c:33
add_depth_to_space_layer
static int add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
Definition: dnn_backend_tf.c:617
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
avassert.h
DNN_BACKEND_COMMON_OPTIONS
#define DNN_BACKEND_COMMON_OPTIONS
Definition: dnn_backend_common.h:31
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
fill_model_input_tf
static int fill_model_input_tf(TFModel *tf_model, TFRequestItem *request)
Definition: dnn_backend_tf.c:947
TFRequestItem::exec_module
DNNAsyncExecModule exec_module
Definition: dnn_backend_tf.c:80
float
float
Definition: af_crystalizer.c:122
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:128
ConvolutionalParams::input_num
int32_t input_num
Definition: dnn_backend_native_layer_conv2d.h:28
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:51
DLT_MAXIMUM
@ DLT_MAXIMUM
Definition: dnn_backend_native.h:45
TANH
@ TANH
Definition: dnn_backend_native.h:55
read_graph
static TF_Buffer * read_graph(const char *model_filename)
Definition: dnn_backend_tf.c:226
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:58
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:42
op
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
Definition: anm.c:76
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNData
Definition: dnn_interface.h:59
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:172
ConvolutionalParams::activation
DNNActivationFunc activation
Definition: dnn_backend_native_layer_conv2d.h:29
dnn_tensorflow_options
static const AVOption dnn_tensorflow_options[]
Definition: dnn_backend_tf.c:85
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:162
LayerPadParams
Definition: dnn_backend_native_layer_pad.h:33
ctx
AVFormatContext * ctx
Definition: movenc.c:48
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:45
arg
const char * arg
Definition: jacosubdec.c:67
dnn_backend_native_layer_depth2space.h
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_proc_from_frame_to_dnn
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:100
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:515
ff_dnn_flush_tf
int ff_dnn_flush_tf(const DNNModel *model)
Definition: dnn_backend_tf.c:1195
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
Layer::params
void * params
Definition: dnn_backend_native.h:66
ff_frame_to_dnn_detect
int ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:254
NULL
#define NULL
Definition: coverity.c:32
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
NativeModel
Definition: dnn_backend_native.h:124
av_opt_set_from_string
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1666
tf_create_inference_request
static TFInferRequest * tf_create_inference_request(void)
Create a TensorFlow inference request.
Definition: dnn_backend_tf.c:137
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:92
TFModel::task_queue
Queue * task_queue
Definition: dnn_backend_tf.c:62
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_tf.c:1041
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:38
TFModel::status
TF_Status * status
Definition: dnn_backend_tf.c:59
tf_free_request
static void tf_free_request(TFInferRequest *request)
Free the contents of TensorFlow inference request.
Definition: dnn_backend_tf.c:109
transpose_perm
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Definition: twinvq.c:621
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
ConvolutionalParams::kernel_size
int32_t kernel_size
Definition: dnn_backend_native_layer_conv2d.h:28
av_cpu_count
int av_cpu_count(void)
Definition: cpu.c:194
options
const OptionDef options[]
test::width
int width
Definition: vc1dsp.c:38
AVIOContext
Bytestream IO Context.
Definition: avio.h:162
DLT_MIRROR_PAD
@ DLT_MIRROR_PAD
Definition: dnn_backend_native.h:44
DnnLayerMaximumParams::val
union DnnLayerMaximumParams::@204 val
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:42
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:46
cpu.h
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:106
size
int size
Definition: twinvq_data.h:10344
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:127
avio.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
TFOptions::nireq
uint32_t nireq
Definition: dnn_backend_tf.c:46
dnn_backend_native_layer_conv2d.h
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
hex_to_data
static int hex_to_data(uint8_t *data, const char *p)
Definition: dnn_backend_tf.c:362
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
NAME_BUFFER_SIZE
#define NAME_BUFFER_SIZE
Definition: dnn_backend_tf.c:493
tf_start_inference
static int tf_start_inference(void *args)
Start synchronous inference for the TensorFlow model.
Definition: dnn_backend_tf.c:158
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:56
DepthToSpaceParams
Definition: dnn_backend_native_layer_depth2space.h:33
dnn_backend_native.h
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_dnn_load_model_tf
DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_tf.c:852
RELU
@ RELU
Definition: dnn_backend_native.h:55
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
avio_closep
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1285
ConvolutionalParams::output_num
int32_t output_num
Definition: dnn_backend_native_layer_conv2d.h:28
add_maximum_layer
static int add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
Definition: dnn_backend_tf.c:699
TFContext::options
TFOptions options
Definition: dnn_backend_tf.c:51
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:271
LayerPadParams::paddings
int32_t paddings[4][2]
Definition: dnn_backend_native_layer_pad.h:34
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
TFRequestItem::status
TF_Status * status
Definition: dnn_backend_tf.c:79
TFInferRequest::output_tensors
TF_Tensor ** output_tensors
Definition: dnn_backend_tf.c:71
TFModel::session
TF_Session * session
Definition: dnn_backend_tf.c:58
TFRequestItem::infer_request
TFInferRequest * infer_request
Definition: dnn_backend_tf.c:77
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:31
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:63
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:76
av_toupper
static av_const int av_toupper(int c)
Locale-independent conversion of ASCII characters to uppercase.
Definition: avstring.h:236
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:264
TFOptions::async
uint8_t async
Definition: dnn_backend_tf.c:45
safe_queue.h
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:41
len
int len
Definition: vorbis_enc_data.h:426
av_calloc
void * av_calloc(size_t nmemb, size_t size)
Definition: mem.c:272
TFInferRequest::tf_input
TF_Output * tf_input
Definition: dnn_backend_tf.c:72
TFContext
Definition: dnn_backend_tf.c:49
DepthToSpaceParams::block_size
int block_size
Definition: dnn_backend_native_layer_depth2space.h:34
ret
ret
Definition: filter_design.txt:187
DNNModel::get_input
int(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
TFModel::model
DNNModel * model
Definition: dnn_backend_tf.c:56
TFModel
Definition: dnn_backend_tf.c:54
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:39
AVFrame::height
int height
Definition: frame.h:397
allocate_input_tensor
static TF_Tensor * allocate_input_tensor(const DNNData *input)
Definition: dnn_backend_tf.c:259
dnn_backend_common.h
TFRequestItem::lltask
LastLevelTaskItem * lltask
Definition: dnn_backend_tf.c:78
defs.h
avio_read
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:641
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:142
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
avio_open
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1220
AVFilterContext
An instance of a filter.
Definition: avfilter.h:408
ff_dnn_free_model_tf
void ff_dnn_free_model_tf(DNNModel **model)
Definition: dnn_backend_tf.c:1225
ff_dnn_start_inference_async
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
Definition: dnn_backend_common.c:111
DNNModel
Definition: dnn_interface.h:84
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:622
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:40
DNNModel::options
const char * options
Definition: dnn_interface.h:88
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
TFOptions
Definition: dnn_backend_tf.c:43
free_buffer
static void free_buffer(void *data, size_t length)
Definition: dnn_backend_tf.c:97
int32_t
int32_t
Definition: audioconvert.c:56
get_output_tf
static int get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_tf.c:317
DNNExecBaseParams
Definition: dnn_interface.h:67
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:43
DNNModel::get_output
int(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
DLT_DEPTH_TO_SPACE
@ DLT_DEPTH_TO_SPACE
Definition: dnn_backend_native.h:43
avstring.h
dnn_backend_native_layer_maximum.h
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
TFRequestItem
Definition: dnn_backend_tf.c:76
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:45
dnn_backend_tf.h
snprintf
#define snprintf
Definition: snprintf.h:34
CLEANUP_ON_ERROR
#define CLEANUP_ON_ERROR(tf_model)
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
TFModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_tf.c:61
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:44
ConvolutionalParams
Definition: dnn_backend_native_layer_conv2d.h:27
add_pad_layer
static int add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
Definition: dnn_backend_tf.c:641
DNNModel::model
void * model
Definition: dnn_interface.h:86
ff_proc_from_dnn_to_frame
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:27
ConvolutionalParams::biases
float * biases
Definition: dnn_backend_native_layer_conv2d.h:34