FFmpeg
dnn_backend_tf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN tensorflow backend implementation.
24  */
25 
26 #include "dnn_backend_tf.h"
27 #include "dnn_backend_native.h"
30 #include "libavformat/avio.h"
31 #include "libavutil/avassert.h"
32 #include "libavutil/avstring.h"
33 #include "libavutil/cpu.h"
34 #include "../internal.h"
37 #include "dnn_io_proc.h"
38 #include "dnn_backend_common.h"
39 #include "safe_queue.h"
40 #include "queue.h"
41 #include <tensorflow/c/c_api.h>
42 
43 typedef struct TFOptions{
44  char *sess_config;
45  uint32_t nireq;
46 } TFOptions;
47 
48 typedef struct TFContext {
49  const AVClass *class;
51 } TFContext;
52 
53 typedef struct TFModel{
56  TF_Graph *graph;
57  TF_Session *session;
58  TF_Status *status;
61 } TFModel;
62 
63 /**
64  * Stores execution parameters for single
65  * call to the TensorFlow C API
66  */
67 typedef struct TFInferRequest {
68  TF_Output *tf_outputs;
69  TF_Tensor **output_tensors;
70  TF_Output *tf_input;
71  TF_Tensor *input_tensor;
73 
74 typedef struct TFRequestItem {
77  // further properties will be added later for async
79 
80 #define OFFSET(x) offsetof(TFContext, x)
81 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
82 static const AVOption dnn_tensorflow_options[] = {
83  { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
85  { NULL }
86 };
87 
88 AVFILTER_DEFINE_CLASS(dnn_tensorflow);
89 
90 static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_queue);
91 
92 static void free_buffer(void *data, size_t length)
93 {
94  av_freep(&data);
95 }
96 
97 static void tf_free_request(TFInferRequest *request)
98 {
99  if (!request)
100  return;
101  if (request->input_tensor) {
102  TF_DeleteTensor(request->input_tensor);
103  request->input_tensor = NULL;
104  }
105  av_freep(&request->tf_input);
106  av_freep(&request->tf_outputs);
107  if (request->output_tensors) {
108  int nb_output = sizeof(*request->output_tensors)/sizeof(request->output_tensors[0]);
109  for (uint32_t i = 0; i < nb_output; ++i) {
110  if (request->output_tensors[i]) {
111  TF_DeleteTensor(request->output_tensors[i]);
112  request->output_tensors[i] = NULL;
113  }
114  }
115  av_freep(&request->output_tensors);
116  }
117 }
118 
120 {
121  TFInferRequest *infer_request = av_malloc(sizeof(TFInferRequest));
122  infer_request->tf_outputs = NULL;
123  infer_request->tf_input = NULL;
124  infer_request->input_tensor = NULL;
125  infer_request->output_tensors = NULL;
126  return infer_request;
127 }
128 
130 {
131  TFModel *tf_model = task->model;
132  TFContext *ctx = &tf_model->ctx;
133  InferenceItem *inference = av_malloc(sizeof(*inference));
134  if (!inference) {
135  av_log(ctx, AV_LOG_ERROR, "Unable to allocate space for InferenceItem\n");
136  return DNN_ERROR;
137  }
138  task->inference_todo = 1;
139  task->inference_done = 0;
140  inference->task = task;
141  if (ff_queue_push_back(inference_queue, inference) < 0) {
142  av_log(ctx, AV_LOG_ERROR, "Failed to push back inference_queue.\n");
143  av_freep(&inference);
144  return DNN_ERROR;
145  }
146  return DNN_SUCCESS;
147 }
148 
149 static TF_Buffer *read_graph(const char *model_filename)
150 {
151  TF_Buffer *graph_buf;
152  unsigned char *graph_data = NULL;
153  AVIOContext *model_file_context;
154  long size, bytes_read;
155 
156  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
157  return NULL;
158  }
159 
160  size = avio_size(model_file_context);
161 
162  graph_data = av_malloc(size);
163  if (!graph_data){
164  avio_closep(&model_file_context);
165  return NULL;
166  }
167  bytes_read = avio_read(model_file_context, graph_data, size);
168  avio_closep(&model_file_context);
169  if (bytes_read != size){
170  av_freep(&graph_data);
171  return NULL;
172  }
173 
174  graph_buf = TF_NewBuffer();
175  graph_buf->data = graph_data;
176  graph_buf->length = size;
177  graph_buf->data_deallocator = free_buffer;
178 
179  return graph_buf;
180 }
181 
182 static TF_Tensor *allocate_input_tensor(const DNNData *input)
183 {
184  TF_DataType dt;
185  size_t size;
186  int64_t input_dims[] = {1, input->height, input->width, input->channels};
187  switch (input->dt) {
188  case DNN_FLOAT:
189  dt = TF_FLOAT;
190  size = sizeof(float);
191  break;
192  case DNN_UINT8:
193  dt = TF_UINT8;
194  size = 1;
195  break;
196  default:
197  av_assert0(!"should not reach here");
198  }
199 
200  return TF_AllocateTensor(dt, input_dims, 4,
201  input_dims[1] * input_dims[2] * input_dims[3] * size);
202 }
203 
204 static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
205 {
206  TFModel *tf_model = model;
207  TFContext *ctx = &tf_model->ctx;
208  TF_Status *status;
209  int64_t dims[4];
210 
211  TF_Output tf_output;
212  tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);
213  if (!tf_output.oper) {
214  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
215  return DNN_ERROR;
216  }
217 
218  tf_output.index = 0;
219  input->dt = TF_OperationOutputType(tf_output);
220  input->order = DCO_RGB;
221 
222  status = TF_NewStatus();
223  TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);
224  if (TF_GetCode(status) != TF_OK){
225  TF_DeleteStatus(status);
226  av_log(ctx, AV_LOG_ERROR, "Failed to get input tensor shape: number of dimension incorrect\n");
227  return DNN_ERROR;
228  }
229  TF_DeleteStatus(status);
230 
231  // currently only NHWC is supported
232  av_assert0(dims[0] == 1 || dims[0] == -1);
233  input->height = dims[1];
234  input->width = dims[2];
235  input->channels = dims[3];
236 
237  return DNN_SUCCESS;
238 }
239 
240 static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height,
241  const char *output_name, int *output_width, int *output_height)
242 {
244  TFModel *tf_model = model;
245  TFContext *ctx = &tf_model->ctx;
246  AVFrame *in_frame = av_frame_alloc();
247  AVFrame *out_frame = NULL;
248  TaskItem task;
249  TFRequestItem *request;
250 
251  if (!in_frame) {
252  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n");
253  ret = DNN_ERROR;
254  goto err;
255  }
256 
257  out_frame = av_frame_alloc();
258  if (!out_frame) {
259  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output frame\n");
260  ret = DNN_ERROR;
261  goto err;
262  }
263 
264  in_frame->width = input_width;
265  in_frame->height = input_height;
266 
267  task.do_ioproc = 0;
268  task.async = 0;
269  task.input_name = input_name;
270  task.in_frame = in_frame;
271  task.output_names = &output_name;
272  task.out_frame = out_frame;
273  task.model = tf_model;
274  task.nb_output = 1;
275 
276  if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) {
277  av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
278  ret = DNN_ERROR;
279  goto err;
280  }
281 
282  request = ff_safe_queue_pop_front(tf_model->request_queue);
283  if (!request) {
284  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
285  ret = DNN_ERROR;
286  goto err;
287  }
288 
289  ret = execute_model_tf(request, tf_model->inference_queue);
290  *output_width = out_frame->width;
291  *output_height = out_frame->height;
292 
293 err:
294  av_frame_free(&out_frame);
295  av_frame_free(&in_frame);
296  return ret;
297 }
298 
299 #define SPACE_CHARS " \t\r\n"
300 static int hex_to_data(uint8_t *data, const char *p)
301 {
302  int c, len, v;
303 
304  len = 0;
305  v = 1;
306  for (;;) {
307  p += strspn(p, SPACE_CHARS);
308  if (*p == '\0')
309  break;
310  c = av_toupper((unsigned char) *p++);
311  if (c >= '0' && c <= '9')
312  c = c - '0';
313  else if (c >= 'A' && c <= 'F')
314  c = c - 'A' + 10;
315  else
316  break;
317  v = (v << 4) | c;
318  if (v & 0x100) {
319  if (data) {
320  data[len] = v;
321  }
322  len++;
323  v = 1;
324  }
325  }
326  return len;
327 }
328 
329 static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
330 {
331  TFContext *ctx = &tf_model->ctx;
332  TF_Buffer *graph_def;
333  TF_ImportGraphDefOptions *graph_opts;
334  TF_SessionOptions *sess_opts;
335  const TF_Operation *init_op;
336  uint8_t *sess_config = NULL;
337  int sess_config_length = 0;
338 
339  // prepare the sess config data
340  if (tf_model->ctx.options.sess_config != NULL) {
341  const char *config;
342  /*
343  tf_model->ctx.options.sess_config is hex to present the serialized proto
344  required by TF_SetConfig below, so we need to first generate the serialized
345  proto in a python script, tools/python/tf_sess_config.py is a script example
346  to generate the configs of sess_config.
347  */
348  if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {
349  av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");
350  return DNN_ERROR;
351  }
352  config = tf_model->ctx.options.sess_config + 2;
353  sess_config_length = hex_to_data(NULL, config);
354 
355  sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);
356  if (!sess_config) {
357  av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");
358  return DNN_ERROR;
359  }
360  if (hex_to_data(sess_config, config) < 0) {
361  av_log(ctx, AV_LOG_ERROR, "failed to convert hex to data\n");
362  return DNN_ERROR;
363  }
364  }
365 
366  graph_def = read_graph(model_filename);
367  if (!graph_def){
368  av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);
369  av_freep(&sess_config);
370  return DNN_ERROR;
371  }
372  tf_model->graph = TF_NewGraph();
373  tf_model->status = TF_NewStatus();
374  graph_opts = TF_NewImportGraphDefOptions();
375  TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
376  TF_DeleteImportGraphDefOptions(graph_opts);
377  TF_DeleteBuffer(graph_def);
378  if (TF_GetCode(tf_model->status) != TF_OK){
379  TF_DeleteGraph(tf_model->graph);
380  TF_DeleteStatus(tf_model->status);
381  av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");
382  av_freep(&sess_config);
383  return DNN_ERROR;
384  }
385 
386  init_op = TF_GraphOperationByName(tf_model->graph, "init");
387  sess_opts = TF_NewSessionOptions();
388 
389  if (sess_config) {
390  TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);
391  av_freep(&sess_config);
392  if (TF_GetCode(tf_model->status) != TF_OK) {
393  TF_DeleteGraph(tf_model->graph);
394  TF_DeleteStatus(tf_model->status);
395  TF_DeleteSessionOptions(sess_opts);
396  av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",
397  tf_model->ctx.options.sess_config);
398  return DNN_ERROR;
399  }
400  }
401 
402  tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
403  TF_DeleteSessionOptions(sess_opts);
404  if (TF_GetCode(tf_model->status) != TF_OK)
405  {
406  TF_DeleteGraph(tf_model->graph);
407  TF_DeleteStatus(tf_model->status);
408  av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");
409  return DNN_ERROR;
410  }
411 
412  // Run initialization operation with name "init" if it is present in graph
413  if (init_op){
414  TF_SessionRun(tf_model->session, NULL,
415  NULL, NULL, 0,
416  NULL, NULL, 0,
417  &init_op, 1, NULL, tf_model->status);
418  if (TF_GetCode(tf_model->status) != TF_OK)
419  {
420  TF_DeleteSession(tf_model->session, tf_model->status);
421  TF_DeleteGraph(tf_model->graph);
422  TF_DeleteStatus(tf_model->status);
423  av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");
424  return DNN_ERROR;
425  }
426  }
427 
428  return DNN_SUCCESS;
429 }
430 
431 #define NAME_BUFFER_SIZE 256
432 
433 static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
434  ConvolutionalParams* params, const int layer)
435 {
436  TFContext *ctx = &tf_model->ctx;
437  TF_Operation *op;
438  TF_OperationDescription *op_desc;
439  TF_Output input;
440  int64_t strides[] = {1, 1, 1, 1};
441  TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
442  int64_t dims[4];
443  int dims_len;
444  char name_buffer[NAME_BUFFER_SIZE];
445  int32_t size;
446 
447  size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
448  input.index = 0;
449 
450  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
451  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
452  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
453  dims[0] = params->output_num;
454  dims[1] = params->kernel_size;
455  dims[2] = params->kernel_size;
456  dims[3] = params->input_num;
457  dims_len = 4;
458  kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
459  memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
460  TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
461  if (TF_GetCode(tf_model->status) != TF_OK){
462  goto err;
463  }
464  op = TF_FinishOperation(op_desc, tf_model->status);
465  if (TF_GetCode(tf_model->status) != TF_OK){
466  goto err;
467  }
468 
469  snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
470  op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
471  input.oper = op;
472  TF_AddInput(op_desc, input);
473  input.oper = transpose_op;
474  TF_AddInput(op_desc, input);
475  TF_SetAttrType(op_desc, "T", TF_FLOAT);
476  TF_SetAttrType(op_desc, "Tperm", TF_INT32);
477  op = TF_FinishOperation(op_desc, tf_model->status);
478  if (TF_GetCode(tf_model->status) != TF_OK){
479  goto err;
480  }
481 
482  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
483  op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
484  input.oper = *cur_op;
485  TF_AddInput(op_desc, input);
486  input.oper = op;
487  TF_AddInput(op_desc, input);
488  TF_SetAttrType(op_desc, "T", TF_FLOAT);
489  TF_SetAttrIntList(op_desc, "strides", strides, 4);
490  TF_SetAttrString(op_desc, "padding", "VALID", 5);
491  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
492  if (TF_GetCode(tf_model->status) != TF_OK){
493  goto err;
494  }
495 
496  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
497  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
498  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
499  dims[0] = params->output_num;
500  dims_len = 1;
501  biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
502  memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
503  TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
504  if (TF_GetCode(tf_model->status) != TF_OK){
505  goto err;
506  }
507  op = TF_FinishOperation(op_desc, tf_model->status);
508  if (TF_GetCode(tf_model->status) != TF_OK){
509  goto err;
510  }
511 
512  snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
513  op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
514  input.oper = *cur_op;
515  TF_AddInput(op_desc, input);
516  input.oper = op;
517  TF_AddInput(op_desc, input);
518  TF_SetAttrType(op_desc, "T", TF_FLOAT);
519  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
520  if (TF_GetCode(tf_model->status) != TF_OK){
521  goto err;
522  }
523 
524  snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
525  switch (params->activation){
526  case RELU:
527  op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
528  break;
529  case TANH:
530  op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
531  break;
532  case SIGMOID:
533  op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
534  break;
535  default:
536  avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
537  return DNN_ERROR;
538  }
539  input.oper = *cur_op;
540  TF_AddInput(op_desc, input);
541  TF_SetAttrType(op_desc, "T", TF_FLOAT);
542  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
543  if (TF_GetCode(tf_model->status) != TF_OK){
544  goto err;
545  }
546 
547  return DNN_SUCCESS;
548 err:
549  TF_DeleteTensor(kernel_tensor);
550  TF_DeleteTensor(biases_tensor);
551  av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
552  return DNN_ERROR;
553 }
554 
555 static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
556  DepthToSpaceParams *params, const int layer)
557 {
558  TFContext *ctx = &tf_model->ctx;
559  TF_OperationDescription *op_desc;
560  TF_Output input;
561  char name_buffer[NAME_BUFFER_SIZE];
562 
563  snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
564  op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
565  input.oper = *cur_op;
566  input.index = 0;
567  TF_AddInput(op_desc, input);
568  TF_SetAttrType(op_desc, "T", TF_FLOAT);
569  TF_SetAttrInt(op_desc, "block_size", params->block_size);
570  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
571  if (TF_GetCode(tf_model->status) != TF_OK){
572  av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);
573  return DNN_ERROR;
574  }
575 
576  return DNN_SUCCESS;
577 }
578 
579 static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
580  LayerPadParams *params, const int layer)
581 {
582  TFContext *ctx = &tf_model->ctx;
583  TF_Operation *op;
584  TF_Tensor *tensor;
585  TF_OperationDescription *op_desc;
586  TF_Output input;
587  int32_t *pads;
588  int64_t pads_shape[] = {4, 2};
589 
590  char name_buffer[NAME_BUFFER_SIZE];
591  snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
592 
593  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
594  TF_SetAttrType(op_desc, "dtype", TF_INT32);
595  tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
596  pads = (int32_t *)TF_TensorData(tensor);
597  pads[0] = params->paddings[0][0];
598  pads[1] = params->paddings[0][1];
599  pads[2] = params->paddings[1][0];
600  pads[3] = params->paddings[1][1];
601  pads[4] = params->paddings[2][0];
602  pads[5] = params->paddings[2][1];
603  pads[6] = params->paddings[3][0];
604  pads[7] = params->paddings[3][1];
605  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
606  if (TF_GetCode(tf_model->status) != TF_OK){
607  TF_DeleteTensor(tensor);
608  av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
609  return DNN_ERROR;
610  }
611  op = TF_FinishOperation(op_desc, tf_model->status);
612  if (TF_GetCode(tf_model->status) != TF_OK){
613  TF_DeleteTensor(tensor);
614  av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
615  return DNN_ERROR;
616  }
617 
618  op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
619  input.oper = *cur_op;
620  input.index = 0;
621  TF_AddInput(op_desc, input);
622  input.oper = op;
623  TF_AddInput(op_desc, input);
624  TF_SetAttrType(op_desc, "T", TF_FLOAT);
625  TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
626  TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
627  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
628  if (TF_GetCode(tf_model->status) != TF_OK){
629  TF_DeleteTensor(tensor);
630  av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
631  return DNN_ERROR;
632  }
633 
634  return DNN_SUCCESS;
635 }
636 
637 static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
638  DnnLayerMaximumParams *params, const int layer)
639 {
640  TFContext *ctx = &tf_model->ctx;
641  TF_Operation *op;
642  TF_Tensor *tensor;
643  TF_OperationDescription *op_desc;
644  TF_Output input;
645  float *y;
646 
647  char name_buffer[NAME_BUFFER_SIZE];
648  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
649 
650  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
651  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
652  tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
653  y = (float *)TF_TensorData(tensor);
654  *y = params->val.y;
655  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
656  if (TF_GetCode(tf_model->status) != TF_OK){
657  TF_DeleteTensor(tensor);
658  av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
659  return DNN_ERROR;
660  }
661  op = TF_FinishOperation(op_desc, tf_model->status);
662  if (TF_GetCode(tf_model->status) != TF_OK){
663  TF_DeleteTensor(tensor);
664  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
665  return DNN_ERROR;
666  }
667 
668  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
669  op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
670  input.oper = *cur_op;
671  input.index = 0;
672  TF_AddInput(op_desc, input);
673  input.oper = op;
674  TF_AddInput(op_desc, input);
675  TF_SetAttrType(op_desc, "T", TF_FLOAT);
676  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
677  if (TF_GetCode(tf_model->status) != TF_OK){
678  TF_DeleteTensor(tensor);
679  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
680  return DNN_ERROR;
681  }
682 
683  return DNN_SUCCESS;
684 }
685 
686 static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
687 {
688  TFContext *ctx = &tf_model->ctx;
689  int32_t layer;
690  TF_OperationDescription *op_desc;
691  TF_Operation *op;
692  TF_Operation *transpose_op;
693  TF_Tensor *tensor = NULL;
694  TF_Output input;
696  int64_t transpose_perm_shape[] = {4};
697  int64_t input_shape[] = {1, -1, -1, -1};
698  DNNReturnType layer_add_res;
699  DNNModel *model = NULL;
700  NativeModel *native_model;
701 
702  model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
703  if (!model){
704  av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
705  return DNN_ERROR;
706  }
707 
708  native_model = model->model;
709  tf_model->graph = TF_NewGraph();
710  tf_model->status = TF_NewStatus();
711 
712 #define CLEANUP_ON_ERROR(tf_model) \
713  { \
714  TF_DeleteTensor(tensor); \
715  TF_DeleteGraph(tf_model->graph); \
716  TF_DeleteStatus(tf_model->status); \
717  av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
718  return DNN_ERROR; \
719  }
720 
721  op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
722  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
723  TF_SetAttrShape(op_desc, "shape", input_shape, 4);
724  op = TF_FinishOperation(op_desc, tf_model->status);
725  if (TF_GetCode(tf_model->status) != TF_OK){
726  CLEANUP_ON_ERROR(tf_model);
727  }
728 
729  op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
730  TF_SetAttrType(op_desc, "dtype", TF_INT32);
731  tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
732  transpose_perm = (int32_t *)TF_TensorData(tensor);
733  transpose_perm[0] = 1;
734  transpose_perm[1] = 2;
735  transpose_perm[2] = 3;
736  transpose_perm[3] = 0;
737  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
738  if (TF_GetCode(tf_model->status) != TF_OK){
739  CLEANUP_ON_ERROR(tf_model);
740  }
741  transpose_op = TF_FinishOperation(op_desc, tf_model->status);
742  if (TF_GetCode(tf_model->status) != TF_OK){
743  CLEANUP_ON_ERROR(tf_model);
744  }
745 
746  for (layer = 0; layer < native_model->layers_num; ++layer){
747  switch (native_model->layers[layer].type){
748  case DLT_INPUT:
749  layer_add_res = DNN_SUCCESS;
750  break;
751  case DLT_CONV2D:
752  layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
753  (ConvolutionalParams *)native_model->layers[layer].params, layer);
754  break;
755  case DLT_DEPTH_TO_SPACE:
756  layer_add_res = add_depth_to_space_layer(tf_model, &op,
757  (DepthToSpaceParams *)native_model->layers[layer].params, layer);
758  break;
759  case DLT_MIRROR_PAD:
760  layer_add_res = add_pad_layer(tf_model, &op,
761  (LayerPadParams *)native_model->layers[layer].params, layer);
762  break;
763  case DLT_MAXIMUM:
764  layer_add_res = add_maximum_layer(tf_model, &op,
765  (DnnLayerMaximumParams *)native_model->layers[layer].params, layer);
766  break;
767  default:
768  CLEANUP_ON_ERROR(tf_model);
769  }
770 
771  if (layer_add_res != DNN_SUCCESS){
772  CLEANUP_ON_ERROR(tf_model);
773  }
774  }
775 
776  op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
777  input.oper = op;
778  input.index = 0;
779  TF_AddInput(op_desc, input);
780  TF_FinishOperation(op_desc, tf_model->status);
781  if (TF_GetCode(tf_model->status) != TF_OK){
782  CLEANUP_ON_ERROR(tf_model);
783  }
784 
785  ff_dnn_free_model_native(&model);
786 
787  return DNN_SUCCESS;
788 }
789 
790 DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
791 {
792  DNNModel *model = NULL;
793  TFModel *tf_model = NULL;
794  TFContext *ctx = NULL;
795 
796  model = av_mallocz(sizeof(DNNModel));
797  if (!model){
798  return NULL;
799  }
800 
801  tf_model = av_mallocz(sizeof(TFModel));
802  if (!tf_model){
803  av_freep(&model);
804  return NULL;
805  }
806  tf_model->model = model;
807  ctx = &tf_model->ctx;
808  ctx->class = &dnn_tensorflow_class;
809 
810  //parse options
812  if (av_opt_set_from_string(ctx, options, NULL, "=", "&") < 0) {
813  av_log(ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
814  goto err;
815  }
816 
817  if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
818  if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
819  goto err;
820  }
821  }
822 
823  if (ctx->options.nireq <= 0) {
824  ctx->options.nireq = av_cpu_count() / 2 + 1;
825  }
826 
827  tf_model->request_queue = ff_safe_queue_create();
828  if (!tf_model->request_queue) {
829  goto err;
830  }
831 
832  for (int i = 0; i < ctx->options.nireq; i++) {
833  TFRequestItem *item = av_mallocz(sizeof(*item));
834  if (!item) {
835  goto err;
836  }
838  if (!item->infer_request) {
839  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for TensorFlow inference request\n");
840  av_freep(&item);
841  goto err;
842  }
843 
844  if (ff_safe_queue_push_back(tf_model->request_queue, item) < 0) {
845  av_freep(&item->infer_request);
846  av_freep(&item);
847  goto err;
848  }
849  }
850 
851  tf_model->inference_queue = ff_queue_create();
852  if (!tf_model->inference_queue) {
853  goto err;
854  }
855 
856  model->model = tf_model;
857  model->get_input = &get_input_tf;
858  model->get_output = &get_output_tf;
859  model->options = options;
860  model->filter_ctx = filter_ctx;
861  model->func_type = func_type;
862 
863  return model;
864 err:
865  ff_dnn_free_model_tf(&model);
866  return NULL;
867 }
868 
870  DNNData input;
871  InferenceItem *inference;
872  TaskItem *task;
873  TFInferRequest *infer_request;
874  TFContext *ctx = &tf_model->ctx;
875 
876  inference = ff_queue_pop_front(tf_model->inference_queue);
877  av_assert0(inference);
878  task = inference->task;
879  request->inference = inference;
880 
881  if (get_input_tf(tf_model, &input, task->input_name) != DNN_SUCCESS) {
882  goto err;
883  }
884 
885  infer_request = request->infer_request;
886  input.height = task->in_frame->height;
887  input.width = task->in_frame->width;
888 
889  infer_request->tf_input = av_malloc(sizeof(TF_Output));
890  if (!infer_request->tf_input) {
891  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
892  goto err;
893  }
894 
895  infer_request->tf_input->oper = TF_GraphOperationByName(tf_model->graph, task->input_name);
896  if (!infer_request->tf_input->oper){
897  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", task->input_name);
898  goto err;
899  }
900  infer_request->tf_input->index = 0;
901 
902  infer_request->input_tensor = allocate_input_tensor(&input);
903  if (!infer_request->input_tensor){
904  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
905  goto err;
906  }
907  input.data = (float *)TF_TensorData(infer_request->input_tensor);
908 
909  switch (tf_model->model->func_type) {
910  case DFT_PROCESS_FRAME:
911  if (task->do_ioproc) {
912  if (tf_model->model->frame_pre_proc != NULL) {
913  tf_model->model->frame_pre_proc(task->in_frame, &input, tf_model->model->filter_ctx);
914  } else {
916  }
917  }
918  break;
921  break;
922  default:
923  avpriv_report_missing_feature(ctx, "model function type %d", tf_model->model->func_type);
924  break;
925  }
926 
927  infer_request->tf_outputs = av_malloc_array(task->nb_output, sizeof(TF_Output));
928  if (infer_request->tf_outputs == NULL) {
929  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n");
930  goto err;
931  }
932 
933  infer_request->output_tensors = av_mallocz_array(task->nb_output, sizeof(*infer_request->output_tensors));
934  if (!infer_request->output_tensors) {
935  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n");
936  goto err;
937  }
938 
939  for (int i = 0; i < task->nb_output; ++i) {
940  infer_request->output_tensors[i] = NULL;
941  infer_request->tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, task->output_names[i]);
942  if (!infer_request->tf_outputs[i].oper) {
943  av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", task->output_names[i]);
944  goto err;
945  }
946  infer_request->tf_outputs[i].index = 0;
947  }
948 
949  return DNN_SUCCESS;
950 err:
951  tf_free_request(infer_request);
952  return DNN_ERROR;
953 }
954 
955 static void infer_completion_callback(void *args) {
956  TFRequestItem *request = args;
957  InferenceItem *inference = request->inference;
958  TaskItem *task = inference->task;
959  DNNData *outputs;
960  TFInferRequest *infer_request = request->infer_request;
961  TFModel *tf_model = task->model;
962  TFContext *ctx = &tf_model->ctx;
963 
964  outputs = av_malloc_array(task->nb_output, sizeof(*outputs));
965  if (!outputs) {
966  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *outputs\n");
967  goto err;
968  }
969 
970  for (uint32_t i = 0; i < task->nb_output; ++i) {
971  outputs[i].height = TF_Dim(infer_request->output_tensors[i], 1);
972  outputs[i].width = TF_Dim(infer_request->output_tensors[i], 2);
973  outputs[i].channels = TF_Dim(infer_request->output_tensors[i], 3);
974  outputs[i].data = TF_TensorData(infer_request->output_tensors[i]);
975  outputs[i].dt = TF_TensorType(infer_request->output_tensors[i]);
976  }
977  switch (tf_model->model->func_type) {
978  case DFT_PROCESS_FRAME:
979  //it only support 1 output if it's frame in & frame out
980  if (task->do_ioproc) {
981  if (tf_model->model->frame_post_proc != NULL) {
982  tf_model->model->frame_post_proc(task->out_frame, outputs, tf_model->model->filter_ctx);
983  } else {
985  }
986  } else {
987  task->out_frame->width = outputs[0].width;
988  task->out_frame->height = outputs[0].height;
989  }
990  break;
992  if (!tf_model->model->detect_post_proc) {
993  av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
994  return;
995  }
996  tf_model->model->detect_post_proc(task->out_frame, outputs, task->nb_output, tf_model->model->filter_ctx);
997  break;
998  default:
999  av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
1000  goto err;
1001  }
1002  task->inference_done++;
1003 err:
1004  tf_free_request(infer_request);
1005  av_freep(&outputs);
1006 
1007  if (ff_safe_queue_push_back(tf_model->request_queue, request) < 0) {
1008  av_freep(&request->infer_request);
1009  av_freep(&request);
1010  av_log(ctx, AV_LOG_ERROR, "Failed to push back request_queue.\n");
1011  }
1012 }
1013 
1014 static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_queue)
1015 {
1016  TFModel *tf_model;
1017  TFContext *ctx;
1018  TFInferRequest *infer_request;
1019  InferenceItem *inference;
1020  TaskItem *task;
1021 
1022  inference = ff_queue_peek_front(inference_queue);
1023  if (!inference) {
1024  av_log(NULL, AV_LOG_ERROR, "Failed to get inference item\n");
1025  return DNN_ERROR;
1026  }
1027  task = inference->task;
1028  tf_model = task->model;
1029  ctx = &tf_model->ctx;
1030 
1031  if (task->async) {
1032  avpriv_report_missing_feature(ctx, "Async execution not supported");
1033  return DNN_ERROR;
1034  } else {
1035  if (fill_model_input_tf(tf_model, request) != DNN_SUCCESS) {
1036  return DNN_ERROR;
1037  }
1038 
1039  infer_request = request->infer_request;
1040  TF_SessionRun(tf_model->session, NULL,
1041  infer_request->tf_input, &infer_request->input_tensor, 1,
1042  infer_request->tf_outputs, infer_request->output_tensors,
1043  task->nb_output, NULL, 0, NULL,
1044  tf_model->status);
1045  if (TF_GetCode(tf_model->status) != TF_OK) {
1046  tf_free_request(infer_request);
1047  av_log(ctx, AV_LOG_ERROR, "Failed to run session when executing model\n");
1048  return DNN_ERROR;
1049  }
1050  infer_completion_callback(request);
1051  return (task->inference_done == task->inference_todo) ? DNN_SUCCESS : DNN_ERROR;
1052  }
1053 }
1054 
1056 {
1057  TFModel *tf_model = model->model;
1058  TFContext *ctx = &tf_model->ctx;
1059  TaskItem task;
1060  TFRequestItem *request;
1061 
1062  if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) {
1063  return DNN_ERROR;
1064  }
1065 
1066  if (ff_dnn_fill_task(&task, exec_params, tf_model, 0, 1) != DNN_SUCCESS) {
1067  return DNN_ERROR;
1068  }
1069 
1070  if (extract_inference_from_task(&task, tf_model->inference_queue) != DNN_SUCCESS) {
1071  av_log(ctx, AV_LOG_ERROR, "unable to extract inference from task.\n");
1072  return DNN_ERROR;
1073  }
1074 
1075  request = ff_safe_queue_pop_front(tf_model->request_queue);
1076  if (!request) {
1077  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
1078  return DNN_ERROR;
1079  }
1080 
1081  return execute_model_tf(request, tf_model->inference_queue);
1082 }
1083 
1085 {
1086  TFModel *tf_model;
1087 
1088  if (*model){
1089  tf_model = (*model)->model;
1090  while (ff_safe_queue_size(tf_model->request_queue) != 0) {
1093  av_freep(&item->infer_request);
1094  av_freep(&item);
1095  }
1097 
1098  while (ff_queue_size(tf_model->inference_queue) != 0) {
1100  av_freep(&item);
1101  }
1102  ff_queue_destroy(tf_model->inference_queue);
1103 
1104  if (tf_model->graph){
1105  TF_DeleteGraph(tf_model->graph);
1106  }
1107  if (tf_model->session){
1108  TF_CloseSession(tf_model->session, tf_model->status);
1109  TF_DeleteSession(tf_model->session, tf_model->status);
1110  }
1111  if (tf_model->status){
1112  TF_DeleteStatus(tf_model->status);
1113  }
1114  av_freep(&tf_model);
1115  av_freep(model);
1116  }
1117 }
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(dnn_tensorflow)
InferenceItem
Definition: dnn_backend_common.h:47
TFOptions::sess_config
char * sess_config
Definition: dnn_backend_tf.c:44
TFInferRequest
Stores execution parameters for single call to the TensorFlow C API.
Definition: dnn_backend_tf.c:67
TFInferRequest::tf_outputs
TF_Output * tf_outputs
Definition: dnn_backend_tf.c:68
FLAGS
#define FLAGS
Definition: dnn_backend_tf.c:81
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
ff_dnn_fill_task
DNNReturnType ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:53
filter_ctx
static FilteringContext * filter_ctx
Definition: transcoding.c:49
TFModel::graph
TF_Graph * graph
Definition: dnn_backend_tf.c:56
av_opt_set_defaults
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
Definition: opt.c:1358
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
TFModel::ctx
TFContext ctx
Definition: dnn_backend_tf.c:54
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:26
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DnnLayerMaximumParams::y
float y
Definition: dnn_backend_native_layer_maximum.h:36
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:112
ConvolutionalParams::kernel
float * kernel
Definition: dnn_backend_native_layer_conv2d.h:33
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:303
AVFrame::width
int width
Definition: frame.h:361
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:116
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
AVOption
AVOption.
Definition: opt.h:247
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
TFInferRequest::input_tensor
TF_Tensor * input_tensor
Definition: dnn_backend_tf.c:71
data
const char data[16]
Definition: mxf.c:143
av_mallocz_array
void * av_mallocz_array(size_t nmemb, size_t size)
Definition: mem.c:196
fill_model_input_tf
static DNNReturnType fill_model_input_tf(TFModel *tf_model, TFRequestItem *request)
Definition: dnn_backend_tf.c:869
dnn_io_proc.h
TFModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_tf.c:59
TaskItem
Definition: dnn_backend_common.h:33
DnnLayerMaximumParams
Definition: dnn_backend_native_layer_maximum.h:33
add_pad_layer
static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
Definition: dnn_backend_tf.c:579
get_output_tf
static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_tf.c:240
avio_size
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:332
tf_sess_config.config
config
Definition: tf_sess_config.py:33
OFFSET
#define OFFSET(x)
Definition: dnn_backend_tf.c:80
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
dnn_backend_native_layer_pad.h
InferenceItem::task
TaskItem * task
Definition: dnn_backend_common.h:48
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
DLT_INPUT
@ DLT_INPUT
Definition: dnn_backend_native.h:40
ff_proc_from_dnn_to_frame
DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:27
ff_proc_from_frame_to_dnn
DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:100
TaskItem::model
void * model
Definition: dnn_backend_common.h:34
DNN_SUCCESS
@ DNN_SUCCESS
Definition: dnn_interface.h:33
DNNModel::get_output
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
get_input_tf
static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_tf.c:204
SIGMOID
@ SIGMOID
Definition: dnn_backend_native.h:54
SPACE_CHARS
#define SPACE_CHARS
Definition: dnn_backend_tf.c:299
Queue
Linear double-ended data structure.
Definition: queue.c:33
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:99
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
avassert.h
extract_inference_from_task
static DNNReturnType extract_inference_from_task(TaskItem *task, Queue *inference_queue)
Definition: dnn_backend_tf.c:129
DNN_BACKEND_COMMON_OPTIONS
#define DNN_BACKEND_COMMON_OPTIONS
Definition: dnn_backend_common.h:29
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:126
ConvolutionalParams::input_num
int32_t input_num
Definition: dnn_backend_native_layer_conv2d.h:28
DLT_MAXIMUM
@ DLT_MAXIMUM
Definition: dnn_backend_native.h:44
TANH
@ TANH
Definition: dnn_backend_native.h:54
read_graph
static TF_Buffer * read_graph(const char *model_filename)
Definition: dnn_backend_tf.c:149
add_maximum_layer
static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
Definition: dnn_backend_tf.c:637
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:57
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:41
op
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
Definition: anm.c:75
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
load_native_model
static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:686
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNReturnType
DNNReturnType
Definition: dnn_interface.h:33
DNNData
Definition: dnn_interface.h:59
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
ConvolutionalParams::activation
DNNActivationFunc activation
Definition: dnn_backend_native_layer_conv2d.h:29
dnn_tensorflow_options
static const AVOption dnn_tensorflow_options[]
Definition: dnn_backend_tf.c:82
DnnLayerMaximumParams::val
union DnnLayerMaximumParams::@198 val
LayerPadParams
Definition: dnn_backend_native_layer_pad.h:33
ctx
AVFormatContext * ctx
Definition: movenc.c:48
DNNModel::get_input
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:42
dnn_backend_native_layer_depth2space.h
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:410
load_tf_model
static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:329
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
Layer::params
void * params
Definition: dnn_backend_native.h:65
NULL
#define NULL
Definition: coverity.c:32
add_conv_layer
static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
Definition: dnn_backend_tf.c:433
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
NativeModel
Definition: dnn_backend_native.h:122
av_opt_set_from_string
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1559
tf_create_inference_request
static TFInferRequest * tf_create_inference_request(void)
Definition: dnn_backend_tf.c:119
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_tf.c:955
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:35
TFModel::status
TF_Status * status
Definition: dnn_backend_tf.c:58
tf_free_request
static void tf_free_request(TFInferRequest *request)
Definition: dnn_backend_tf.c:97
transpose_perm
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Definition: twinvq.c:620
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
ConvolutionalParams::kernel_size
int32_t kernel_size
Definition: dnn_backend_native_layer_conv2d.h:28
av_cpu_count
int av_cpu_count(void)
Definition: cpu.c:185
options
const OptionDef options[]
AVIOContext
Bytestream IO Context.
Definition: avio.h:161
DLT_MIRROR_PAD
@ DLT_MIRROR_PAD
Definition: dnn_backend_native.h:43
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:39
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:43
cpu.h
queue.h
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:106
ff_dnn_execute_model_tf
DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_tf.c:1055
size
int size
Definition: twinvq_data.h:10344
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:125
avio.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
TFOptions::nireq
uint32_t nireq
Definition: dnn_backend_tf.c:45
dnn_backend_native_layer_conv2d.h
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
hex_to_data
static int hex_to_data(uint8_t *data, const char *p)
Definition: dnn_backend_tf.c:300
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
NAME_BUFFER_SIZE
#define NAME_BUFFER_SIZE
Definition: dnn_backend_tf.c:431
ff_frame_to_dnn_detect
DNNReturnType ff_frame_to_dnn_detect(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:252
DepthToSpaceParams
Definition: dnn_backend_native_layer_depth2space.h:33
dnn_backend_native.h
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_dnn_load_model_tf
DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_tf.c:790
RELU
@ RELU
Definition: dnn_backend_native.h:54
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
avio_closep
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1175
ConvolutionalParams::output_num
int32_t output_num
Definition: dnn_backend_native_layer_conv2d.h:28
i
int i
Definition: input.c:406
TFContext::options
TFOptions options
Definition: dnn_backend_tf.c:50
LayerPadParams::paddings
int32_t paddings[4][2]
Definition: dnn_backend_native_layer_pad.h:34
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
TFInferRequest::output_tensors
TF_Tensor ** output_tensors
Definition: dnn_backend_tf.c:69
TFModel::session
TF_Session * session
Definition: dnn_backend_tf.c:57
TFRequestItem::infer_request
TFInferRequest * infer_request
Definition: dnn_backend_tf.c:75
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
av_toupper
static av_const int av_toupper(int c)
Locale-independent conversion of ASCII characters to uppercase.
Definition: avstring.h:236
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:243
safe_queue.h
TaskItem::output_names
const char ** output_names
Definition: dnn_backend_common.h:38
len
int len
Definition: vorbis_enc_data.h:426
TFInferRequest::tf_input
TF_Output * tf_input
Definition: dnn_backend_tf.c:70
TFContext
Definition: dnn_backend_tf.c:48
DepthToSpaceParams::block_size
int block_size
Definition: dnn_backend_native_layer_depth2space.h:34
ret
ret
Definition: filter_design.txt:187
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
TFModel::model
DNNModel * model
Definition: dnn_backend_tf.c:55
TFModel
Definition: dnn_backend_tf.c:53
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: defs.h:40
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:36
AVFrame::height
int height
Definition: frame.h:361
allocate_input_tensor
static TF_Tensor * allocate_input_tensor(const DNNData *input)
Definition: dnn_backend_tf.c:182
execute_model_tf
static DNNReturnType execute_model_tf(TFRequestItem *request, Queue *inference_queue)
Definition: dnn_backend_tf.c:1014
add_depth_to_space_layer
static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
Definition: dnn_backend_tf.c:555
dnn_backend_common.h
avio_read
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:616
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
avio_open
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1120
AVFilterContext
An instance of a filter.
Definition: avfilter.h:333
ff_dnn_free_model_tf
void ff_dnn_free_model_tf(DNNModel **model)
Definition: dnn_backend_tf.c:1084
DNNModel
Definition: dnn_interface.h:84
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:655
TFRequestItem::inference
InferenceItem * inference
Definition: dnn_backend_tf.c:76
TaskItem::input_name
const char * input_name
Definition: dnn_backend_common.h:37
TFModel::inference_queue
Queue * inference_queue
Definition: dnn_backend_tf.c:60
DNNModel::options
const char * options
Definition: dnn_interface.h:88
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
TFOptions
Definition: dnn_backend_tf.c:43
free_buffer
static void free_buffer(void *data, size_t length)
Definition: dnn_backend_tf.c:92
int32_t
int32_t
Definition: audioconvert.c:56
DNNExecBaseParams
Definition: dnn_interface.h:67
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:40
DLT_DEPTH_TO_SPACE
@ DLT_DEPTH_TO_SPACE
Definition: dnn_backend_native.h:42
avstring.h
dnn_backend_native_layer_maximum.h
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:228
TFRequestItem
Definition: dnn_backend_tf.c:74
dnn_backend_tf.h
snprintf
#define snprintf
Definition: snprintf.h:34
CLEANUP_ON_ERROR
#define CLEANUP_ON_ERROR(tf_model)
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
TaskItem::nb_output
uint32_t nb_output
Definition: dnn_backend_common.h:41
ConvolutionalParams
Definition: dnn_backend_native_layer_conv2d.h:27
DNNModel::model
void * model
Definition: dnn_interface.h:86
ConvolutionalParams::biases
float * biases
Definition: dnn_backend_native_layer_conv2d.h:34