FFmpeg
dnn_backend_tf.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2018 Sergey Lavrushkin
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN tensorflow backend implementation.
24  */
25 
26 #include "dnn_backend_tf.h"
27 #include "dnn_backend_native.h"
28 #include "dnn_backend_native_layer_conv2d.h"
29 #include "dnn_backend_native_layer_depth2space.h"
30 #include "libavformat/avio.h"
31 #include "libavutil/avassert.h"
32 #include "libavutil/avstring.h"
33 #include "../internal.h"
34 #include "dnn_backend_native_layer_pad.h"
35 #include "dnn_backend_native_layer_maximum.h"
36 #include "dnn_io_proc.h"
37 #include "dnn_backend_common.h"
38 #include <tensorflow/c/c_api.h>
39 
40 typedef struct TFOptions{
41  char *sess_config;
42 } TFOptions;
43 
44 typedef struct TFContext {
45  const AVClass *class;
46  TFOptions options;
47 } TFContext;
48 
49 typedef struct TFModel{
50  TFContext ctx;
51  DNNModel *model;
52  TF_Graph *graph;
53  TF_Session *session;
54  TF_Status *status;
55 } TFModel;
56 
57 #define OFFSET(x) offsetof(TFContext, x)
58 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
59 static const AVOption dnn_tensorflow_options[] = {
60  { "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
61  { NULL }
62 };
63 
64 AVFILTER_DEFINE_CLASS(dnn_tensorflow);
65 
66 static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
67  const char **output_names, uint32_t nb_output, AVFrame *out_frame,
68  int do_ioproc);
69 
70 static void free_buffer(void *data, size_t length)
71 {
72  av_freep(&data);
73 }
74 
75 static TF_Buffer *read_graph(const char *model_filename)
76 {
77  TF_Buffer *graph_buf;
78  unsigned char *graph_data = NULL;
79  AVIOContext *model_file_context;
80  long size, bytes_read;
81 
82  if (avio_open(&model_file_context, model_filename, AVIO_FLAG_READ) < 0){
83  return NULL;
84  }
85 
86  size = avio_size(model_file_context);
87 
88  graph_data = av_malloc(size);
89  if (!graph_data){
90  avio_closep(&model_file_context);
91  return NULL;
92  }
93  bytes_read = avio_read(model_file_context, graph_data, size);
94  avio_closep(&model_file_context);
95  if (bytes_read != size){
96  av_freep(&graph_data);
97  return NULL;
98  }
99 
100  graph_buf = TF_NewBuffer();
101  graph_buf->data = graph_data;
102  graph_buf->length = size;
103  graph_buf->data_deallocator = free_buffer;
104 
105  return graph_buf;
106 }
107 
108 static TF_Tensor *allocate_input_tensor(const DNNData *input)
109 {
110  TF_DataType dt;
111  size_t size;
112  int64_t input_dims[] = {1, input->height, input->width, input->channels};
113  switch (input->dt) {
114  case DNN_FLOAT:
115  dt = TF_FLOAT;
116  size = sizeof(float);
117  break;
118  case DNN_UINT8:
119  dt = TF_UINT8;
120  size = 1;
121  break;
122  default:
123  av_assert0(!"should not reach here");
124  }
125 
126  return TF_AllocateTensor(dt, input_dims, 4,
127  input_dims[1] * input_dims[2] * input_dims[3] * size);
128 }
129 
130 static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
131 {
132  TFModel *tf_model = model;
133  TFContext *ctx = &tf_model->ctx;
134  TF_Status *status;
135  int64_t dims[4];
136 
137  TF_Output tf_output;
138  tf_output.oper = TF_GraphOperationByName(tf_model->graph, input_name);
139  if (!tf_output.oper) {
140  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
141  return DNN_ERROR;
142  }
143 
144  tf_output.index = 0;
145  input->dt = TF_OperationOutputType(tf_output);
146  input->order = DCO_RGB;
147 
148  status = TF_NewStatus();
149  TF_GraphGetTensorShape(tf_model->graph, tf_output, dims, 4, status);
150  if (TF_GetCode(status) != TF_OK){
151  TF_DeleteStatus(status);
152  av_log(ctx, AV_LOG_ERROR, "Failed to get input tensor shape: number of dimension incorrect\n");
153  return DNN_ERROR;
154  }
155  TF_DeleteStatus(status);
156 
157  // currently only NHWC is supported
158  av_assert0(dims[0] == 1 || dims[0] == -1);
159  input->height = dims[1];
160  input->width = dims[2];
161  input->channels = dims[3];
162 
163  return DNN_SUCCESS;
164 }
165 
166 static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height,
167  const char *output_name, int *output_width, int *output_height)
168 {
169  DNNReturnType ret;
170  TFModel *tf_model = model;
171  TFContext *ctx = &tf_model->ctx;
172  AVFrame *in_frame = av_frame_alloc();
173  AVFrame *out_frame = NULL;
174 
175  if (!in_frame) {
176  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input frame\n");
177  return DNN_ERROR;
178  }
179 
180  out_frame = av_frame_alloc();
181  if (!out_frame) {
182  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output frame\n");
183  av_frame_free(&in_frame);
184  return DNN_ERROR;
185  }
186 
187  in_frame->width = input_width;
188  in_frame->height = input_height;
189 
190  ret = execute_model_tf(tf_model->model, input_name, in_frame, &output_name, 1, out_frame, 0);
191  *output_width = out_frame->width;
192  *output_height = out_frame->height;
193 
194  av_frame_free(&out_frame);
195  av_frame_free(&in_frame);
196  return ret;
197 }
198 
199 #define SPACE_CHARS " \t\r\n"
200 static int hex_to_data(uint8_t *data, const char *p)
201 {
202  int c, len, v;
203 
204  len = 0;
205  v = 1;
206  for (;;) {
207  p += strspn(p, SPACE_CHARS);
208  if (*p == '\0')
209  break;
210  c = av_toupper((unsigned char) *p++);
211  if (c >= '0' && c <= '9')
212  c = c - '0';
213  else if (c >= 'A' && c <= 'F')
214  c = c - 'A' + 10;
215  else
216  break;
217  v = (v << 4) | c;
218  if (v & 0x100) {
219  if (data) {
220  data[len] = v;
221  }
222  len++;
223  v = 1;
224  }
225  }
226  return len;
227 }
228 
229 static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
230 {
231  TFContext *ctx = &tf_model->ctx;
232  TF_Buffer *graph_def;
233  TF_ImportGraphDefOptions *graph_opts;
234  TF_SessionOptions *sess_opts;
235  const TF_Operation *init_op;
236  uint8_t *sess_config = NULL;
237  int sess_config_length = 0;
238 
239  // prepare the sess config data
240  if (tf_model->ctx.options.sess_config != NULL) {
241  const char *config;
242  /*
243  tf_model->ctx.options.sess_config is hex to present the serialized proto
244  required by TF_SetConfig below, so we need to first generate the serialized
245  proto in a python script, tools/python/tf_sess_config.py is a script example
246  to generate the configs of sess_config.
247  */
248  if (strncmp(tf_model->ctx.options.sess_config, "0x", 2) != 0) {
249  av_log(ctx, AV_LOG_ERROR, "sess_config should start with '0x'\n");
250  return DNN_ERROR;
251  }
252  config = tf_model->ctx.options.sess_config + 2;
253  sess_config_length = hex_to_data(NULL, config);
254 
255  sess_config = av_mallocz(sess_config_length + AV_INPUT_BUFFER_PADDING_SIZE);
256  if (!sess_config) {
257  av_log(ctx, AV_LOG_ERROR, "failed to allocate memory\n");
258  return DNN_ERROR;
259  }
260  if (hex_to_data(sess_config, config) < 0) {
261  av_log(ctx, AV_LOG_ERROR, "failed to convert hex to data\n");
262  return DNN_ERROR;
263  }
264  }
265 
266  graph_def = read_graph(model_filename);
267  if (!graph_def){
268  av_log(ctx, AV_LOG_ERROR, "Failed to read model \"%s\" graph\n", model_filename);
269  av_freep(&sess_config);
270  return DNN_ERROR;
271  }
272  tf_model->graph = TF_NewGraph();
273  tf_model->status = TF_NewStatus();
274  graph_opts = TF_NewImportGraphDefOptions();
275  TF_GraphImportGraphDef(tf_model->graph, graph_def, graph_opts, tf_model->status);
276  TF_DeleteImportGraphDefOptions(graph_opts);
277  TF_DeleteBuffer(graph_def);
278  if (TF_GetCode(tf_model->status) != TF_OK){
279  TF_DeleteGraph(tf_model->graph);
280  TF_DeleteStatus(tf_model->status);
281  av_log(ctx, AV_LOG_ERROR, "Failed to import serialized graph to model graph\n");
282  av_freep(&sess_config);
283  return DNN_ERROR;
284  }
285 
286  init_op = TF_GraphOperationByName(tf_model->graph, "init");
287  sess_opts = TF_NewSessionOptions();
288 
289  if (sess_config) {
290  TF_SetConfig(sess_opts, sess_config, sess_config_length,tf_model->status);
291  av_freep(&sess_config);
292  if (TF_GetCode(tf_model->status) != TF_OK) {
293  TF_DeleteGraph(tf_model->graph);
294  TF_DeleteStatus(tf_model->status);
295  TF_DeleteSessionOptions(sess_opts);
296  av_log(ctx, AV_LOG_ERROR, "Failed to set config for sess options with %s\n",
297  tf_model->ctx.options.sess_config);
298  return DNN_ERROR;
299  }
300  }
301 
302  tf_model->session = TF_NewSession(tf_model->graph, sess_opts, tf_model->status);
303  TF_DeleteSessionOptions(sess_opts);
304  if (TF_GetCode(tf_model->status) != TF_OK)
305  {
306  TF_DeleteGraph(tf_model->graph);
307  TF_DeleteStatus(tf_model->status);
308  av_log(ctx, AV_LOG_ERROR, "Failed to create new session with model graph\n");
309  return DNN_ERROR;
310  }
311 
312  // Run initialization operation with name "init" if it is present in graph
313  if (init_op){
314  TF_SessionRun(tf_model->session, NULL,
315  NULL, NULL, 0,
316  NULL, NULL, 0,
317  &init_op, 1, NULL, tf_model->status);
318  if (TF_GetCode(tf_model->status) != TF_OK)
319  {
320  TF_DeleteSession(tf_model->session, tf_model->status);
321  TF_DeleteGraph(tf_model->graph);
322  TF_DeleteStatus(tf_model->status);
323  av_log(ctx, AV_LOG_ERROR, "Failed to run session when initializing\n");
324  return DNN_ERROR;
325  }
326  }
327 
328  return DNN_SUCCESS;
329 }
330 
331 #define NAME_BUFFER_SIZE 256
332 
333 static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op,
334  ConvolutionalParams* params, const int layer)
335 {
336  TFContext *ctx = &tf_model->ctx;
337  TF_Operation *op;
338  TF_OperationDescription *op_desc;
339  TF_Output input;
340  int64_t strides[] = {1, 1, 1, 1};
341  TF_Tensor *kernel_tensor = NULL, *biases_tensor = NULL;
342  int64_t dims[4];
343  int dims_len;
344  char name_buffer[NAME_BUFFER_SIZE];
345  int32_t size;
346 
347  size = params->input_num * params->output_num * params->kernel_size * params->kernel_size;
348  input.index = 0;
349 
350  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_kernel%d", layer);
351  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
352  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
353  dims[0] = params->output_num;
354  dims[1] = params->kernel_size;
355  dims[2] = params->kernel_size;
356  dims[3] = params->input_num;
357  dims_len = 4;
358  kernel_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, size * sizeof(float));
359  memcpy(TF_TensorData(kernel_tensor), params->kernel, size * sizeof(float));
360  TF_SetAttrTensor(op_desc, "value", kernel_tensor, tf_model->status);
361  if (TF_GetCode(tf_model->status) != TF_OK){
362  goto err;
363  }
364  op = TF_FinishOperation(op_desc, tf_model->status);
365  if (TF_GetCode(tf_model->status) != TF_OK){
366  goto err;
367  }
368 
369  snprintf(name_buffer, NAME_BUFFER_SIZE, "transpose%d", layer);
370  op_desc = TF_NewOperation(tf_model->graph, "Transpose", name_buffer);
371  input.oper = op;
372  TF_AddInput(op_desc, input);
373  input.oper = transpose_op;
374  TF_AddInput(op_desc, input);
375  TF_SetAttrType(op_desc, "T", TF_FLOAT);
376  TF_SetAttrType(op_desc, "Tperm", TF_INT32);
377  op = TF_FinishOperation(op_desc, tf_model->status);
378  if (TF_GetCode(tf_model->status) != TF_OK){
379  goto err;
380  }
381 
382  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv2d%d", layer);
383  op_desc = TF_NewOperation(tf_model->graph, "Conv2D", name_buffer);
384  input.oper = *cur_op;
385  TF_AddInput(op_desc, input);
386  input.oper = op;
387  TF_AddInput(op_desc, input);
388  TF_SetAttrType(op_desc, "T", TF_FLOAT);
389  TF_SetAttrIntList(op_desc, "strides", strides, 4);
390  TF_SetAttrString(op_desc, "padding", "VALID", 5);
391  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
392  if (TF_GetCode(tf_model->status) != TF_OK){
393  goto err;
394  }
395 
396  snprintf(name_buffer, NAME_BUFFER_SIZE, "conv_biases%d", layer);
397  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
398  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
399  dims[0] = params->output_num;
400  dims_len = 1;
401  biases_tensor = TF_AllocateTensor(TF_FLOAT, dims, dims_len, params->output_num * sizeof(float));
402  memcpy(TF_TensorData(biases_tensor), params->biases, params->output_num * sizeof(float));
403  TF_SetAttrTensor(op_desc, "value", biases_tensor, tf_model->status);
404  if (TF_GetCode(tf_model->status) != TF_OK){
405  goto err;
406  }
407  op = TF_FinishOperation(op_desc, tf_model->status);
408  if (TF_GetCode(tf_model->status) != TF_OK){
409  goto err;
410  }
411 
412  snprintf(name_buffer, NAME_BUFFER_SIZE, "bias_add%d", layer);
413  op_desc = TF_NewOperation(tf_model->graph, "BiasAdd", name_buffer);
414  input.oper = *cur_op;
415  TF_AddInput(op_desc, input);
416  input.oper = op;
417  TF_AddInput(op_desc, input);
418  TF_SetAttrType(op_desc, "T", TF_FLOAT);
419  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
420  if (TF_GetCode(tf_model->status) != TF_OK){
421  goto err;
422  }
423 
424  snprintf(name_buffer, NAME_BUFFER_SIZE, "activation%d", layer);
425  switch (params->activation){
426  case RELU:
427  op_desc = TF_NewOperation(tf_model->graph, "Relu", name_buffer);
428  break;
429  case TANH:
430  op_desc = TF_NewOperation(tf_model->graph, "Tanh", name_buffer);
431  break;
432  case SIGMOID:
433  op_desc = TF_NewOperation(tf_model->graph, "Sigmoid", name_buffer);
434  break;
435  default:
436  avpriv_report_missing_feature(ctx, "convolutional activation function %d", params->activation);
437  return DNN_ERROR;
438  }
439  input.oper = *cur_op;
440  TF_AddInput(op_desc, input);
441  TF_SetAttrType(op_desc, "T", TF_FLOAT);
442  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
443  if (TF_GetCode(tf_model->status) != TF_OK){
444  goto err;
445  }
446 
447  return DNN_SUCCESS;
448 err:
449  TF_DeleteTensor(kernel_tensor);
450  TF_DeleteTensor(biases_tensor);
451  av_log(ctx, AV_LOG_ERROR, "Failed to add conv layer %d\n", layer);
452  return DNN_ERROR;
453 }
454 
455 static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op,
456  DepthToSpaceParams *params, const int layer)
457 {
458  TFContext *ctx = &tf_model->ctx;
459  TF_OperationDescription *op_desc;
460  TF_Output input;
461  char name_buffer[NAME_BUFFER_SIZE];
462 
463  snprintf(name_buffer, NAME_BUFFER_SIZE, "depth_to_space%d", layer);
464  op_desc = TF_NewOperation(tf_model->graph, "DepthToSpace", name_buffer);
465  input.oper = *cur_op;
466  input.index = 0;
467  TF_AddInput(op_desc, input);
468  TF_SetAttrType(op_desc, "T", TF_FLOAT);
469  TF_SetAttrInt(op_desc, "block_size", params->block_size);
470  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
471  if (TF_GetCode(tf_model->status) != TF_OK){
472  av_log(ctx, AV_LOG_ERROR, "Failed to add depth_to_space to layer %d\n", layer);
473  return DNN_ERROR;
474  }
475 
476  return DNN_SUCCESS;
477 }
478 
479 static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op,
480  LayerPadParams *params, const int layer)
481 {
482  TFContext *ctx = &tf_model->ctx;
483  TF_Operation *op;
484  TF_Tensor *tensor;
485  TF_OperationDescription *op_desc;
486  TF_Output input;
487  int32_t *pads;
488  int64_t pads_shape[] = {4, 2};
489 
490  char name_buffer[NAME_BUFFER_SIZE];
491  snprintf(name_buffer, NAME_BUFFER_SIZE, "pad%d", layer);
492 
493  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
494  TF_SetAttrType(op_desc, "dtype", TF_INT32);
495  tensor = TF_AllocateTensor(TF_INT32, pads_shape, 2, 4 * 2 * sizeof(int32_t));
496  pads = (int32_t *)TF_TensorData(tensor);
497  pads[0] = params->paddings[0][0];
498  pads[1] = params->paddings[0][1];
499  pads[2] = params->paddings[1][0];
500  pads[3] = params->paddings[1][1];
501  pads[4] = params->paddings[2][0];
502  pads[5] = params->paddings[2][1];
503  pads[6] = params->paddings[3][0];
504  pads[7] = params->paddings[3][1];
505  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
506  if (TF_GetCode(tf_model->status) != TF_OK){
507  TF_DeleteTensor(tensor);
508  av_log(ctx, AV_LOG_ERROR, "Failed to set value for pad of layer %d\n", layer);
509  return DNN_ERROR;
510  }
511  op = TF_FinishOperation(op_desc, tf_model->status);
512  if (TF_GetCode(tf_model->status) != TF_OK){
513  TF_DeleteTensor(tensor);
514  av_log(ctx, AV_LOG_ERROR, "Failed to add pad to layer %d\n", layer);
515  return DNN_ERROR;
516  }
517 
518  op_desc = TF_NewOperation(tf_model->graph, "MirrorPad", "mirror_pad");
519  input.oper = *cur_op;
520  input.index = 0;
521  TF_AddInput(op_desc, input);
522  input.oper = op;
523  TF_AddInput(op_desc, input);
524  TF_SetAttrType(op_desc, "T", TF_FLOAT);
525  TF_SetAttrType(op_desc, "Tpaddings", TF_INT32);
526  TF_SetAttrString(op_desc, "mode", "SYMMETRIC", 9);
527  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
528  if (TF_GetCode(tf_model->status) != TF_OK){
529  TF_DeleteTensor(tensor);
530  av_log(ctx, AV_LOG_ERROR, "Failed to add mirror_pad to layer %d\n", layer);
531  return DNN_ERROR;
532  }
533 
534  return DNN_SUCCESS;
535 }
536 
537 static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op,
538  DnnLayerMaximumParams *params, const int layer)
539 {
540  TFContext *ctx = &tf_model->ctx;
541  TF_Operation *op;
542  TF_Tensor *tensor;
543  TF_OperationDescription *op_desc;
544  TF_Output input;
545  float *y;
546 
547  char name_buffer[NAME_BUFFER_SIZE];
548  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum/y%d", layer);
549 
550  op_desc = TF_NewOperation(tf_model->graph, "Const", name_buffer);
551  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
552  tensor = TF_AllocateTensor(TF_FLOAT, NULL, 0, TF_DataTypeSize(TF_FLOAT));
553  y = (float *)TF_TensorData(tensor);
554  *y = params->val.y;
555  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
556  if (TF_GetCode(tf_model->status) != TF_OK){
557  TF_DeleteTensor(tensor);
558  av_log(ctx, AV_LOG_ERROR, "Failed to set value for maximum/y of layer %d", layer);
559  return DNN_ERROR;
560  }
561  op = TF_FinishOperation(op_desc, tf_model->status);
562  if (TF_GetCode(tf_model->status) != TF_OK){
563  TF_DeleteTensor(tensor);
564  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum/y to layer %d\n", layer);
565  return DNN_ERROR;
566  }
567 
568  snprintf(name_buffer, NAME_BUFFER_SIZE, "maximum%d", layer);
569  op_desc = TF_NewOperation(tf_model->graph, "Maximum", name_buffer);
570  input.oper = *cur_op;
571  input.index = 0;
572  TF_AddInput(op_desc, input);
573  input.oper = op;
574  TF_AddInput(op_desc, input);
575  TF_SetAttrType(op_desc, "T", TF_FLOAT);
576  *cur_op = TF_FinishOperation(op_desc, tf_model->status);
577  if (TF_GetCode(tf_model->status) != TF_OK){
578  TF_DeleteTensor(tensor);
579  av_log(ctx, AV_LOG_ERROR, "Failed to add maximum to layer %d\n", layer);
580  return DNN_ERROR;
581  }
582 
583  return DNN_SUCCESS;
584 }
585 
586 static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
587 {
588  TFContext *ctx = &tf_model->ctx;
589  int32_t layer;
590  TF_OperationDescription *op_desc;
591  TF_Operation *op;
592  TF_Operation *transpose_op;
593  TF_Tensor *tensor = NULL;
594  TF_Output input;
595  int32_t *transpose_perm;
596  int64_t transpose_perm_shape[] = {4};
597  int64_t input_shape[] = {1, -1, -1, -1};
598  DNNReturnType layer_add_res;
599  DNNModel *model = NULL;
600  NativeModel *native_model;
601 
602  model = ff_dnn_load_model_native(model_filename, DFT_PROCESS_FRAME, NULL, NULL);
603  if (!model){
604  av_log(ctx, AV_LOG_ERROR, "Failed to load native model\n");
605  return DNN_ERROR;
606  }
607 
608  native_model = model->model;
609  tf_model->graph = TF_NewGraph();
610  tf_model->status = TF_NewStatus();
611 
612 #define CLEANUP_ON_ERROR(tf_model) \
613  { \
614  TF_DeleteTensor(tensor); \
615  TF_DeleteGraph(tf_model->graph); \
616  TF_DeleteStatus(tf_model->status); \
617  av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
618  return DNN_ERROR; \
619  }
620 
621  op_desc = TF_NewOperation(tf_model->graph, "Placeholder", "x");
622  TF_SetAttrType(op_desc, "dtype", TF_FLOAT);
623  TF_SetAttrShape(op_desc, "shape", input_shape, 4);
624  op = TF_FinishOperation(op_desc, tf_model->status);
625  if (TF_GetCode(tf_model->status) != TF_OK){
626  CLEANUP_ON_ERROR(tf_model);
627  }
628 
629  op_desc = TF_NewOperation(tf_model->graph, "Const", "transpose_perm");
630  TF_SetAttrType(op_desc, "dtype", TF_INT32);
631  tensor = TF_AllocateTensor(TF_INT32, transpose_perm_shape, 1, 4 * sizeof(int32_t));
632  transpose_perm = (int32_t *)TF_TensorData(tensor);
633  transpose_perm[0] = 1;
634  transpose_perm[1] = 2;
635  transpose_perm[2] = 3;
636  transpose_perm[3] = 0;
637  TF_SetAttrTensor(op_desc, "value", tensor, tf_model->status);
638  if (TF_GetCode(tf_model->status) != TF_OK){
639  CLEANUP_ON_ERROR(tf_model);
640  }
641  transpose_op = TF_FinishOperation(op_desc, tf_model->status);
642  if (TF_GetCode(tf_model->status) != TF_OK){
643  CLEANUP_ON_ERROR(tf_model);
644  }
645 
646  for (layer = 0; layer < native_model->layers_num; ++layer){
647  switch (native_model->layers[layer].type){
648  case DLT_INPUT:
649  layer_add_res = DNN_SUCCESS;
650  break;
651  case DLT_CONV2D:
652  layer_add_res = add_conv_layer(tf_model, transpose_op, &op,
653  (ConvolutionalParams *)native_model->layers[layer].params, layer);
654  break;
655  case DLT_DEPTH_TO_SPACE:
656  layer_add_res = add_depth_to_space_layer(tf_model, &op,
657  (DepthToSpaceParams *)native_model->layers[layer].params, layer);
658  break;
659  case DLT_MIRROR_PAD:
660  layer_add_res = add_pad_layer(tf_model, &op,
661  (LayerPadParams *)native_model->layers[layer].params, layer);
662  break;
663  case DLT_MAXIMUM:
664  layer_add_res = add_maximum_layer(tf_model, &op,
665  (DnnLayerMaximumParams *)native_model->layers[layer].params, layer);
666  break;
667  default:
668  CLEANUP_ON_ERROR(tf_model);
669  }
670 
671  if (layer_add_res != DNN_SUCCESS){
672  CLEANUP_ON_ERROR(tf_model);
673  }
674  }
675 
676  op_desc = TF_NewOperation(tf_model->graph, "Identity", "y");
677  input.oper = op;
678  input.index = 0;
679  TF_AddInput(op_desc, input);
680  TF_FinishOperation(op_desc, tf_model->status);
681  if (TF_GetCode(tf_model->status) != TF_OK){
682  CLEANUP_ON_ERROR(tf_model);
683  }
684 
685  ff_dnn_free_model_native(&model);
686 
687  return DNN_SUCCESS;
688 }
689 
690 DNNModel *ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
691 {
692  DNNModel *model = NULL;
693  TFModel *tf_model = NULL;
694 
695  model = av_mallocz(sizeof(DNNModel));
696  if (!model){
697  return NULL;
698  }
699 
700  tf_model = av_mallocz(sizeof(TFModel));
701  if (!tf_model){
702  av_freep(&model);
703  return NULL;
704  }
705  tf_model->ctx.class = &dnn_tensorflow_class;
706  tf_model->model = model;
707 
708  //parse options
709  av_opt_set_defaults(&tf_model->ctx);
710  if (av_opt_set_from_string(&tf_model->ctx, options, NULL, "=", "&") < 0) {
711  av_log(&tf_model->ctx, AV_LOG_ERROR, "Failed to parse options \"%s\"\n", options);
712  av_freep(&tf_model);
713  av_freep(&model);
714  return NULL;
715  }
716 
717  if (load_tf_model(tf_model, model_filename) != DNN_SUCCESS){
718  if (load_native_model(tf_model, model_filename) != DNN_SUCCESS){
719  av_freep(&tf_model);
720  av_freep(&model);
721 
722  return NULL;
723  }
724  }
725 
726  model->model = tf_model;
727  model->get_input = &get_input_tf;
728  model->get_output = &get_output_tf;
729  model->options = options;
730  model->filter_ctx = filter_ctx;
731  model->func_type = func_type;
732 
733  return model;
734 }
735 
736 static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame,
737  const char **output_names, uint32_t nb_output, AVFrame *out_frame,
738  int do_ioproc)
739 {
740  TF_Output *tf_outputs;
741  TFModel *tf_model = model->model;
742  TFContext *ctx = &tf_model->ctx;
743  DNNData input, *outputs;
744  TF_Tensor **output_tensors;
745  TF_Output tf_input;
746  TF_Tensor *input_tensor;
747 
748  if (get_input_tf(tf_model, &input, input_name) != DNN_SUCCESS)
749  return DNN_ERROR;
750  input.height = in_frame->height;
751  input.width = in_frame->width;
752 
753  tf_input.oper = TF_GraphOperationByName(tf_model->graph, input_name);
754  if (!tf_input.oper){
755  av_log(ctx, AV_LOG_ERROR, "Could not find \"%s\" in model\n", input_name);
756  return DNN_ERROR;
757  }
758  tf_input.index = 0;
759  input_tensor = allocate_input_tensor(&input);
760  if (!input_tensor){
761  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for input tensor\n");
762  return DNN_ERROR;
763  }
764  input.data = (float *)TF_TensorData(input_tensor);
765 
766  if (do_ioproc) {
767  if (tf_model->model->frame_pre_proc != NULL) {
768  tf_model->model->frame_pre_proc(in_frame, &input, tf_model->model->filter_ctx);
769  } else {
770  ff_proc_from_frame_to_dnn(in_frame, &input, tf_model->model->func_type, ctx);
771  }
772  }
773 
774  tf_outputs = av_malloc_array(nb_output, sizeof(*tf_outputs));
775  if (tf_outputs == NULL) {
776  TF_DeleteTensor(input_tensor);
777  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *tf_outputs\n"); \
778  return DNN_ERROR;
779  }
780 
781  output_tensors = av_mallocz_array(nb_output, sizeof(*output_tensors));
782  if (!output_tensors) {
783  TF_DeleteTensor(input_tensor);
784  av_freep(&tf_outputs);
785  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for output tensor\n"); \
786  return DNN_ERROR;
787  }
788 
789  for (int i = 0; i < nb_output; ++i) {
790  tf_outputs[i].oper = TF_GraphOperationByName(tf_model->graph, output_names[i]);
791  if (!tf_outputs[i].oper) {
792  TF_DeleteTensor(input_tensor);
793  av_freep(&tf_outputs);
794  av_freep(&output_tensors);
795  av_log(ctx, AV_LOG_ERROR, "Could not find output \"%s\" in model\n", output_names[i]); \
796  return DNN_ERROR;
797  }
798  tf_outputs[i].index = 0;
799  }
800 
801  TF_SessionRun(tf_model->session, NULL,
802  &tf_input, &input_tensor, 1,
803  tf_outputs, output_tensors, nb_output,
804  NULL, 0, NULL, tf_model->status);
805  if (TF_GetCode(tf_model->status) != TF_OK) {
806  TF_DeleteTensor(input_tensor);
807  av_freep(&tf_outputs);
808  av_freep(&output_tensors);
809  av_log(ctx, AV_LOG_ERROR, "Failed to run session when executing model\n");
810  return DNN_ERROR;
811  }
812 
813  outputs = av_malloc_array(nb_output, sizeof(*outputs));
814  if (!outputs) {
815  TF_DeleteTensor(input_tensor);
816  av_freep(&tf_outputs);
817  av_freep(&output_tensors);
818  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for *outputs\n"); \
819  return DNN_ERROR;
820  }
821 
822  for (uint32_t i = 0; i < nb_output; ++i) {
823  outputs[i].height = TF_Dim(output_tensors[i], 1);
824  outputs[i].width = TF_Dim(output_tensors[i], 2);
825  outputs[i].channels = TF_Dim(output_tensors[i], 3);
826  outputs[i].data = TF_TensorData(output_tensors[i]);
827  outputs[i].dt = TF_TensorType(output_tensors[i]);
828  }
829  switch (model->func_type) {
830  case DFT_PROCESS_FRAME:
831  //it only support 1 output if it's frame in & frame out
832  if (do_ioproc) {
833  if (tf_model->model->frame_post_proc != NULL) {
834  tf_model->model->frame_post_proc(out_frame, outputs, tf_model->model->filter_ctx);
835  } else {
836  ff_proc_from_dnn_to_frame(out_frame, outputs, ctx);
837  }
838  } else {
839  out_frame->width = outputs[0].width;
840  out_frame->height = outputs[0].height;
841  }
842  break;
843  case DFT_ANALYTICS_DETECT:
844  if (!model->detect_post_proc) {
845  av_log(ctx, AV_LOG_ERROR, "Detect filter needs provide post proc\n");
846  return DNN_ERROR;
847  }
848  model->detect_post_proc(out_frame, outputs, nb_output, model->filter_ctx);
849  break;
850  default:
851  for (uint32_t i = 0; i < nb_output; ++i) {
852  if (output_tensors[i]) {
853  TF_DeleteTensor(output_tensors[i]);
854  }
855  }
856  TF_DeleteTensor(input_tensor);
857  av_freep(&output_tensors);
858  av_freep(&tf_outputs);
859  av_freep(&outputs);
860 
861  av_log(ctx, AV_LOG_ERROR, "Tensorflow backend does not support this kind of dnn filter now\n");
862  return DNN_ERROR;
863  }
864 
865  for (uint32_t i = 0; i < nb_output; ++i) {
866  if (output_tensors[i]) {
867  TF_DeleteTensor(output_tensors[i]);
868  }
869  }
870  TF_DeleteTensor(input_tensor);
871  av_freep(&output_tensors);
872  av_freep(&tf_outputs);
873  av_freep(&outputs);
874  return DNN_SUCCESS;
875 }
876 
877 DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
878 {
879  TFModel *tf_model = model->model;
880  TFContext *ctx = &tf_model->ctx;
881 
882  if (ff_check_exec_params(ctx, DNN_TF, model->func_type, exec_params) != 0) {
883  return DNN_ERROR;
884  }
885 
886  return execute_model_tf(model, exec_params->input_name, exec_params->in_frame,
887  exec_params->output_names, exec_params->nb_output, exec_params->out_frame, 1);
888 }
889 
890 void ff_dnn_free_model_tf(DNNModel **model)
891 {
892  TFModel *tf_model;
893 
894  if (*model){
895  tf_model = (*model)->model;
896  if (tf_model->graph){
897  TF_DeleteGraph(tf_model->graph);
898  }
899  if (tf_model->session){
900  TF_CloseSession(tf_model->session, tf_model->status);
901  TF_DeleteSession(tf_model->session, tf_model->status);
902  }
903  if (tf_model->status){
904  TF_DeleteStatus(tf_model->status);
905  }
906  av_freep(&tf_model);
907  av_freep(model);
908  }
909 }
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(dnn_tensorflow)
TFOptions::sess_config
char * sess_config
Definition: dnn_backend_tf.c:41
FLAGS
#define FLAGS
Definition: dnn_backend_tf.c:58
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
filter_ctx
static FilteringContext * filter_ctx
Definition: transcoding.c:48
TFModel::graph
TF_Graph * graph
Definition: dnn_backend_tf.c:52
av_opt_set_defaults
void av_opt_set_defaults(void *s)
Set the values of all AVOption fields to their default values.
Definition: opt.c:1358
TFModel::ctx
TFContext ctx
Definition: dnn_backend_tf.c:50
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:52
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:26
DnnLayerMaximumParams::y
float y
Definition: dnn_backend_native_layer_maximum.h:36
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:111
ConvolutionalParams::kernel
float * kernel
Definition: dnn_backend_native_layer_conv2d.h:33
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:303
AVFrame::width
int width
Definition: frame.h:361
ff_dnn_load_model_native
DNNModel * ff_dnn_load_model_native(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_native.c:116
AVOption
AVOption.
Definition: opt.h:248
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:101
data
const char data[16]
Definition: mxf.c:142
av_mallocz_array
void * av_mallocz_array(size_t nmemb, size_t size)
Definition: mem.c:196
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:68
dnn_io_proc.h
DnnLayerMaximumParams
Definition: dnn_backend_native_layer_maximum.h:33
add_pad_layer
static DNNReturnType add_pad_layer(TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
Definition: dnn_backend_tf.c:479
get_output_tf
static DNNReturnType get_output_tf(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_tf.c:166
DNNExecBaseParams::in_frame
AVFrame * in_frame
Definition: dnn_interface.h:71
avio_size
int64_t avio_size(AVIOContext *s)
Get the filesize.
Definition: aviobuf.c:332
tf_sess_config.config
config
Definition: tf_sess_config.py:33
OFFSET
#define OFFSET(x)
Definition: dnn_backend_tf.c:57
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
dnn_backend_native_layer_pad.h
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:90
DLT_INPUT
@ DLT_INPUT
Definition: dnn_backend_native.h:40
ff_proc_from_dnn_to_frame
DNNReturnType ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:27
DNN_SUCCESS
@ DNN_SUCCESS
Definition: dnn_interface.h:33
DNNModel::get_output
DNNReturnType(* get_output)(void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:97
get_input_tf
static DNNReturnType get_input_tf(void *model, DNNData *input, const char *input_name)
Definition: dnn_backend_tf.c:130
SIGMOID
@ SIGMOID
Definition: dnn_backend_native.h:54
SPACE_CHARS
#define SPACE_CHARS
Definition: dnn_backend_tf.c:199
av_frame_alloc
AVFrame * av_frame_alloc(void)
Allocate an AVFrame and set its fields to default values.
Definition: frame.c:98
avassert.h
DNN_TF
@ DNN_TF
Definition: dnn_interface.h:35
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:181
DnnLayerMaximumParams::val
union DnnLayerMaximumParams::@202 val
NativeModel::layers_num
int32_t layers_num
Definition: dnn_backend_native.h:126
ConvolutionalParams::input_num
int32_t input_num
Definition: dnn_backend_native_layer_conv2d.h:28
DLT_MAXIMUM
@ DLT_MAXIMUM
Definition: dnn_backend_native.h:44
TANH
@ TANH
Definition: dnn_backend_native.h:54
read_graph
static TF_Buffer * read_graph(const char *model_filename)
Definition: dnn_backend_tf.c:75
add_maximum_layer
static DNNReturnType add_maximum_layer(TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
Definition: dnn_backend_tf.c:537
Layer::type
DNNLayerType type
Definition: dnn_backend_native.h:57
DLT_CONV2D
@ DLT_CONV2D
Definition: dnn_backend_native.h:41
op
static int op(uint8_t **dst, const uint8_t *dst_end, GetByteContext *gb, int pixel, int count, int *x, int width, int linesize)
Perform decode operation.
Definition: anm.c:75
load_native_model
static DNNReturnType load_native_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:586
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DNNReturnType
DNNReturnType
Definition: dnn_interface.h:33
DNNData
Definition: dnn_interface.h:59
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
ConvolutionalParams::activation
DNNActivationFunc activation
Definition: dnn_backend_native_layer_conv2d.h:29
dnn_tensorflow_options
static const AVOption dnn_tensorflow_options[]
Definition: dnn_backend_tf.c:59
LayerPadParams
Definition: dnn_backend_native_layer_pad.h:33
ctx
AVFormatContext * ctx
Definition: movenc.c:48
DNNModel::get_input
DNNReturnType(* get_input)(void *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:95
dnn_backend_native_layer_depth2space.h
ff_dnn_free_model_native
void ff_dnn_free_model_native(DNNModel **model)
Definition: dnn_backend_native.c:410
load_tf_model
static DNNReturnType load_tf_model(TFModel *tf_model, const char *model_filename)
Definition: dnn_backend_tf.c:229
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:67
Layer::params
void * params
Definition: dnn_backend_native.h:65
NULL
#define NULL
Definition: coverity.c:32
add_conv_layer
static DNNReturnType add_conv_layer(TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
Definition: dnn_backend_tf.c:333
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:104
NativeModel
Definition: dnn_backend_native.h:122
av_opt_set_from_string
int av_opt_set_from_string(void *ctx, const char *opts, const char *const *shorthand, const char *key_val_sep, const char *pairs_sep)
Parse the key-value pairs list in opts.
Definition: opt.c:1559
TFModel::status
TF_Status * status
Definition: dnn_backend_tf.c:54
execute_model_tf
static DNNReturnType execute_model_tf(const DNNModel *model, const char *input_name, AVFrame *in_frame, const char **output_names, uint32_t nb_output, AVFrame *out_frame, int do_ioproc)
Definition: dnn_backend_tf.c:736
transpose_perm
static void transpose_perm(int16_t *out, int16_t *in, int num_vect, const uint8_t line_len[2], int length_div)
Interpret the input data as in the following table:
Definition: twinvq.c:620
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
ConvolutionalParams::kernel_size
int32_t kernel_size
Definition: dnn_backend_native_layer_conv2d.h:28
options
const OptionDef options[]
AVIOContext
Bytestream IO Context.
Definition: avio.h:161
DLT_MIRROR_PAD
@ DLT_MIRROR_PAD
Definition: dnn_backend_native.h:43
TFContext::class
const AVClass * class
Definition: dnn_backend_tf.c:45
DNNModel::detect_post_proc
DetectPostProc detect_post_proc
Definition: dnn_interface.h:106
ff_dnn_execute_model_tf
DNNReturnType ff_dnn_execute_model_tf(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_tf.c:877
size
int size
Definition: twinvq_data.h:10344
NativeModel::layers
Layer * layers
Definition: dnn_backend_native.h:125
avio.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:92
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
dnn_backend_native_layer_conv2d.h
hex_to_data
static int hex_to_data(uint8_t *data, const char *p)
Definition: dnn_backend_tf.c:200
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:37
NAME_BUFFER_SIZE
#define NAME_BUFFER_SIZE
Definition: dnn_backend_tf.c:331
DepthToSpaceParams
Definition: dnn_backend_native_layer_depth2space.h:33
dnn_backend_native.h
DNNExecBaseParams::out_frame
AVFrame * out_frame
Definition: dnn_interface.h:72
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_proc_from_frame_to_dnn
DNNReturnType ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, DNNFunctionType func_type, void *log_ctx)
Definition: dnn_io_proc.c:275
ff_dnn_load_model_tf
DNNModel * ff_dnn_load_model_tf(const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
Definition: dnn_backend_tf.c:690
RELU
@ RELU
Definition: dnn_backend_native.h:54
avio_closep
int avio_closep(AVIOContext **s)
Close the resource accessed by the AVIOContext *s, free it and set the pointer pointing to it to NULL...
Definition: aviobuf.c:1175
ConvolutionalParams::output_num
int32_t output_num
Definition: dnn_backend_native_layer_conv2d.h:28
i
int i
Definition: input.c:407
TFContext::options
TFOptions options
Definition: dnn_backend_tf.c:46
LayerPadParams::paddings
int32_t paddings[4][2]
Definition: dnn_backend_native_layer_pad.h:34
DFT_ANALYTICS_DETECT
@ DFT_ANALYTICS_DETECT
Definition: dnn_interface.h:55
TFModel::session
TF_Session * session
Definition: dnn_backend_tf.c:53
av_malloc_array
#define av_malloc_array(a, b)
Definition: tableprint_vlc.h:32
av_toupper
static av_const int av_toupper(int c)
Locale-independent conversion of ASCII characters to uppercase.
Definition: avstring.h:236
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:243
len
int len
Definition: vorbis_enc_data.h:452
TFContext
Definition: dnn_backend_tf.c:44
DepthToSpaceParams::block_size
int block_size
Definition: dnn_backend_native_layer_depth2space.h:34
ret
ret
Definition: filter_design.txt:187
DNN_UINT8
@ DNN_UINT8
Definition: dnn_interface.h:37
TFModel::model
DNNModel * model
Definition: dnn_backend_tf.c:51
TFModel
Definition: dnn_backend_tf.c:49
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Definition: avcodec.h:192
AVFrame::height
int height
Definition: frame.h:361
allocate_input_tensor
static TF_Tensor * allocate_input_tensor(const DNNData *input)
Definition: dnn_backend_tf.c:108
add_depth_to_space_layer
static DNNReturnType add_depth_to_space_layer(TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
Definition: dnn_backend_tf.c:455
dnn_backend_common.h
avio_read
int avio_read(AVIOContext *s, unsigned char *buf, int size)
Read size bytes from AVIOContext into buf.
Definition: aviobuf.c:616
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:42
DNNExecBaseParams::output_names
const char ** output_names
Definition: dnn_interface.h:69
avio_open
int avio_open(AVIOContext **s, const char *url, int flags)
Create and initialize a AVIOContext for accessing the resource indicated by url.
Definition: aviobuf.c:1120
AVFilterContext
An instance of a filter.
Definition: avfilter.h:333
ff_dnn_free_model_tf
void ff_dnn_free_model_tf(DNNModel **model)
Definition: dnn_backend_tf.c:890
DNNModel
Definition: dnn_interface.h:84
AVIO_FLAG_READ
#define AVIO_FLAG_READ
read-only
Definition: avio.h:674
DNNModel::options
const char * options
Definition: dnn_interface.h:88
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
TFOptions
Definition: dnn_backend_tf.c:40
free_buffer
static void free_buffer(void *data, size_t length)
Definition: dnn_backend_tf.c:70
int32_t
int32_t
Definition: audioconvert.c:56
DNNExecBaseParams
Definition: dnn_interface.h:67
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
DLT_DEPTH_TO_SPACE
@ DLT_DEPTH_TO_SPACE
Definition: dnn_backend_native.h:42
avstring.h
dnn_backend_native_layer_maximum.h
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:229
dnn_backend_tf.h
snprintf
#define snprintf
Definition: snprintf.h:34
CLEANUP_ON_ERROR
#define CLEANUP_ON_ERROR(tf_model)
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:54
ConvolutionalParams
Definition: dnn_backend_native_layer_conv2d.h:27
DNNExecBaseParams::nb_output
uint32_t nb_output
Definition: dnn_interface.h:70
DNNModel::model
void * model
Definition: dnn_interface.h:86
ConvolutionalParams::biases
float * biases
Definition: dnn_backend_native_layer_conv2d.h:34
Generated on Tue May 11 2021 19:21:44 for FFmpeg by   doxygen 1.8.17