FFmpeg
Data Structures | Macros | Functions | Variables
dnn_backend_tf.c File Reference
#include "dnn_backend_tf.h"
#include "dnn_backend_native.h"
#include "dnn_backend_native_layer_conv2d.h"
#include "dnn_backend_native_layer_depth2space.h"
#include "libavformat/avio.h"
#include "libavutil/avassert.h"
#include "libavutil/avstring.h"
#include "libavutil/cpu.h"
#include "libavcodec/defs.h"
#include "../internal.h"
#include "dnn_backend_native_layer_pad.h"
#include "dnn_backend_native_layer_maximum.h"
#include "dnn_io_proc.h"
#include "dnn_backend_common.h"
#include "safe_queue.h"
#include <tensorflow/c/c_api.h>

Go to the source code of this file.

Data Structures

struct  TFOptions
 
struct  TFContext
 
struct  TFModel
 
struct  TFInferRequest
 Stores execution parameters for single call to the TensorFlow C API. More...
 
struct  TFRequestItem
 

Macros

#define OFFSET(x)   offsetof(TFContext, x)
 
#define FLAGS   AV_OPT_FLAG_FILTERING_PARAM
 
#define SPACE_CHARS   " \t\r\n"
 
#define NAME_BUFFER_SIZE   256
 
#define CLEANUP_ON_ERROR(tf_model)
 

Functions

 AVFILTER_DEFINE_CLASS (dnn_tensorflow)
 
static DNNReturnType execute_model_tf (TFRequestItem *request, Queue *lltask_queue)
 
static void infer_completion_callback (void *args)
 
static void destroy_request_item (TFRequestItem **arg)
 Free the TFRequestItem completely. More...
 
static void free_buffer (void *data, size_t length)
 
static void tf_free_request (TFInferRequest *request)
 Free the contents of TensorFlow inference request. More...
 
static TFInferRequesttf_create_inference_request (void)
 Create a TensorFlow inference request. More...
 
static DNNReturnType tf_start_inference (void *args)
 Start synchronous inference for the TensorFlow model. More...
 
static DNNReturnType extract_lltask_from_task (TaskItem *task, Queue *lltask_queue)
 
static TF_Buffer * read_graph (const char *model_filename)
 
static TF_Tensor * allocate_input_tensor (const DNNData *input)
 
static DNNReturnType get_input_tf (void *model, DNNData *input, const char *input_name)
 
static DNNReturnType get_output_tf (void *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
 
static int hex_to_data (uint8_t *data, const char *p)
 
static DNNReturnType load_tf_model (TFModel *tf_model, const char *model_filename)
 
static DNNReturnType add_conv_layer (TFModel *tf_model, TF_Operation *transpose_op, TF_Operation **cur_op, ConvolutionalParams *params, const int layer)
 
static DNNReturnType add_depth_to_space_layer (TFModel *tf_model, TF_Operation **cur_op, DepthToSpaceParams *params, const int layer)
 
static DNNReturnType add_pad_layer (TFModel *tf_model, TF_Operation **cur_op, LayerPadParams *params, const int layer)
 
static DNNReturnType add_maximum_layer (TFModel *tf_model, TF_Operation **cur_op, DnnLayerMaximumParams *params, const int layer)
 
static DNNReturnType load_native_model (TFModel *tf_model, const char *model_filename)
 
DNNModelff_dnn_load_model_tf (const char *model_filename, DNNFunctionType func_type, const char *options, AVFilterContext *filter_ctx)
 
static DNNReturnType fill_model_input_tf (TFModel *tf_model, TFRequestItem *request)
 
DNNReturnType ff_dnn_execute_model_tf (const DNNModel *model, DNNExecBaseParams *exec_params)
 
DNNAsyncStatusType ff_dnn_get_result_tf (const DNNModel *model, AVFrame **in, AVFrame **out)
 
DNNReturnType ff_dnn_flush_tf (const DNNModel *model)
 
void ff_dnn_free_model_tf (DNNModel **model)
 

Variables

static const AVOption dnn_tensorflow_options []
 

Detailed Description

DNN tensorflow backend implementation.

Definition in file dnn_backend_tf.c.

Macro Definition Documentation

◆ OFFSET

#define OFFSET (   x)    offsetof(TFContext, x)

Definition at line 83 of file dnn_backend_tf.c.

◆ FLAGS

#define FLAGS   AV_OPT_FLAG_FILTERING_PARAM

Definition at line 84 of file dnn_backend_tf.c.

◆ SPACE_CHARS

#define SPACE_CHARS   " \t\r\n"

Definition at line 359 of file dnn_backend_tf.c.

◆ NAME_BUFFER_SIZE

#define NAME_BUFFER_SIZE   256

Definition at line 491 of file dnn_backend_tf.c.

◆ CLEANUP_ON_ERROR

#define CLEANUP_ON_ERROR (   tf_model)
Value:
{ \
TF_DeleteTensor(tensor); \
TF_DeleteGraph(tf_model->graph); \
TF_DeleteStatus(tf_model->status); \
av_log(ctx, AV_LOG_ERROR, "Failed to set value or add operator to layer\n"); \
return DNN_ERROR; \
}

Function Documentation

◆ AVFILTER_DEFINE_CLASS()

AVFILTER_DEFINE_CLASS ( dnn_tensorflow  )

◆ execute_model_tf()

static DNNReturnType execute_model_tf ( TFRequestItem request,
Queue lltask_queue 
)
static

Definition at line 1089 of file dnn_backend_tf.c.

Referenced by ff_dnn_execute_model_tf(), and get_output_tf().

◆ infer_completion_callback()

static void infer_completion_callback ( void *  args)
static

Definition at line 1031 of file dnn_backend_tf.c.

Referenced by execute_model_tf(), and ff_dnn_load_model_tf().

◆ destroy_request_item()

static void destroy_request_item ( TFRequestItem **  arg)
inlinestatic

Free the TFRequestItem completely.

Parameters
argAddress of the TFInferRequest instance.

Definition at line 191 of file dnn_backend_tf.c.

Referenced by execute_model_tf(), ff_dnn_flush_tf(), ff_dnn_free_model_tf(), ff_dnn_load_model_tf(), infer_completion_callback(), and tf_start_inference().

◆ free_buffer()

static void free_buffer ( void *  data,
size_t  length 
)
static

Definition at line 97 of file dnn_backend_tf.c.

Referenced by read_graph().

◆ tf_free_request()

static void tf_free_request ( TFInferRequest request)
static

Free the contents of TensorFlow inference request.

It does not free the TFInferRequest instance.

Parameters
requestpointer to TFInferRequest instance. NULL pointer is allowed.

Definition at line 109 of file dnn_backend_tf.c.

Referenced by destroy_request_item(), execute_model_tf(), fill_model_input_tf(), infer_completion_callback(), and tf_start_inference().

◆ tf_create_inference_request()

static TFInferRequest* tf_create_inference_request ( void  )
static

Create a TensorFlow inference request.

All properties are initially unallocated and set as NULL.

Returns
pointer to the allocated TFInferRequest instance.

Definition at line 137 of file dnn_backend_tf.c.

Referenced by ff_dnn_load_model_tf().

◆ tf_start_inference()

static DNNReturnType tf_start_inference ( void *  args)
static

Start synchronous inference for the TensorFlow model.

Parameters
requestpointer to the TFRequestItem for inference
Return values
DNN_SUCCESSif execution is successful
DNN_ERRORif execution fails

Definition at line 157 of file dnn_backend_tf.c.

Referenced by execute_model_tf(), and ff_dnn_load_model_tf().

◆ extract_lltask_from_task()

static DNNReturnType extract_lltask_from_task ( TaskItem task,
Queue lltask_queue 
)
static

Definition at line 205 of file dnn_backend_tf.c.

Referenced by ff_dnn_execute_model_tf(), and get_output_tf().

◆ read_graph()

static TF_Buffer* read_graph ( const char *  model_filename)
static

Definition at line 225 of file dnn_backend_tf.c.

Referenced by load_tf_model().

◆ allocate_input_tensor()

static TF_Tensor* allocate_input_tensor ( const DNNData input)
static

Definition at line 258 of file dnn_backend_tf.c.

Referenced by fill_model_input_tf().

◆ get_input_tf()

static DNNReturnType get_input_tf ( void *  model,
DNNData input,
const char *  input_name 
)
static

Definition at line 280 of file dnn_backend_tf.c.

Referenced by ff_dnn_load_model_tf(), and fill_model_input_tf().

◆ get_output_tf()

static DNNReturnType get_output_tf ( void *  model,
const char *  input_name,
int  input_width,
int  input_height,
const char *  output_name,
int output_width,
int output_height 
)
static

Definition at line 316 of file dnn_backend_tf.c.

Referenced by ff_dnn_load_model_tf().

◆ hex_to_data()

static int hex_to_data ( uint8_t *  data,
const char *  p 
)
static

Definition at line 360 of file dnn_backend_tf.c.

Referenced by load_tf_model().

◆ load_tf_model()

static DNNReturnType load_tf_model ( TFModel tf_model,
const char *  model_filename 
)
static

Definition at line 389 of file dnn_backend_tf.c.

Referenced by ff_dnn_load_model_tf().

◆ add_conv_layer()

static DNNReturnType add_conv_layer ( TFModel tf_model,
TF_Operation *  transpose_op,
TF_Operation **  cur_op,
ConvolutionalParams params,
const int  layer 
)
static

Definition at line 493 of file dnn_backend_tf.c.

Referenced by load_native_model().

◆ add_depth_to_space_layer()

static DNNReturnType add_depth_to_space_layer ( TFModel tf_model,
TF_Operation **  cur_op,
DepthToSpaceParams params,
const int  layer 
)
static

Definition at line 615 of file dnn_backend_tf.c.

Referenced by load_native_model().

◆ add_pad_layer()

static DNNReturnType add_pad_layer ( TFModel tf_model,
TF_Operation **  cur_op,
LayerPadParams params,
const int  layer 
)
static

Definition at line 639 of file dnn_backend_tf.c.

Referenced by load_native_model().

◆ add_maximum_layer()

static DNNReturnType add_maximum_layer ( TFModel tf_model,
TF_Operation **  cur_op,
DnnLayerMaximumParams params,
const int  layer 
)
static

Definition at line 697 of file dnn_backend_tf.c.

Referenced by load_native_model().

◆ load_native_model()

static DNNReturnType load_native_model ( TFModel tf_model,
const char *  model_filename 
)
static

Definition at line 746 of file dnn_backend_tf.c.

Referenced by ff_dnn_load_model_tf().

◆ ff_dnn_load_model_tf()

DNNModel* ff_dnn_load_model_tf ( const char *  model_filename,
DNNFunctionType  func_type,
const char *  options,
AVFilterContext filter_ctx 
)

Definition at line 850 of file dnn_backend_tf.c.

Referenced by ff_get_dnn_module().

◆ fill_model_input_tf()

static DNNReturnType fill_model_input_tf ( TFModel tf_model,
TFRequestItem request 
)
static

Definition at line 945 of file dnn_backend_tf.c.

Referenced by execute_model_tf(), and ff_dnn_flush_tf().

◆ ff_dnn_execute_model_tf()

DNNReturnType ff_dnn_execute_model_tf ( const DNNModel model,
DNNExecBaseParams exec_params 
)

Definition at line 1130 of file dnn_backend_tf.c.

Referenced by ff_get_dnn_module().

◆ ff_dnn_get_result_tf()

DNNAsyncStatusType ff_dnn_get_result_tf ( const DNNModel model,
AVFrame **  in,
AVFrame **  out 
)

Definition at line 1171 of file dnn_backend_tf.c.

Referenced by ff_get_dnn_module().

◆ ff_dnn_flush_tf()

DNNReturnType ff_dnn_flush_tf ( const DNNModel model)

Definition at line 1177 of file dnn_backend_tf.c.

Referenced by ff_get_dnn_module().

◆ ff_dnn_free_model_tf()

void ff_dnn_free_model_tf ( DNNModel **  model)

Definition at line 1207 of file dnn_backend_tf.c.

Referenced by ff_dnn_load_model_tf(), and ff_get_dnn_module().

Variable Documentation

◆ dnn_tensorflow_options

const AVOption dnn_tensorflow_options[]
static
Initial value:
= {
{ "sess_config", "config for SessionOptions", OFFSET(options.sess_config), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, FLAGS },
{ NULL }
}

Definition at line 85 of file dnn_backend_tf.c.

FLAGS
#define FLAGS
Definition: dnn_backend_tf.c:84
OFFSET
#define OFFSET(x)
Definition: dnn_backend_tf.c:83
DNN_BACKEND_COMMON_OPTIONS
#define DNN_BACKEND_COMMON_OPTIONS
Definition: dnn_backend_common.h:31
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
ctx
AVFormatContext * ctx
Definition: movenc.c:48
NULL
#define NULL
Definition: coverity.c:32
options
const OptionDef options[]
DNN_ERROR
@ DNN_ERROR
Definition: dnn_interface.h:33
AV_OPT_TYPE_STRING
@ AV_OPT_TYPE_STRING
Definition: opt.h:228