FFmpeg
dnn_backend_native_layer_mathbinary.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2020
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN native backend implementation.
24  */
25 
26 #include "dnn_backend_native.h"
28 
29 typedef float (*FunType)(float src0, float src1);
30 
31 static float sub(float src0, float src1)
32 {
33  return src0 - src1;
34 }
35 static float add(float src0, float src1)
36 {
37  return src0 + src1;
38 }
39 static float mul(float src0, float src1)
40 {
41  return src0 * src1;
42 }
43 static float realdiv(float src0, float src1)
44 {
45  return src0 / src1;
46 }
47 static float minimum(float src0, float src1)
48 {
49  return FFMIN(src0, src1);
50 }
51 static float floormod(float src0, float src1)
52 {
53  return (float)((int)(src0) % (int)(src1));
54 }
55 
56 static void math_binary_commutative(FunType pfun, const DnnLayerMathBinaryParams *params, const DnnOperand *input, DnnOperand *output, DnnOperand *operands, const int32_t *input_operand_indexes)
57 {
58  int dims_count;
59  const float *src;
60  float *dst;
62  src = input->data;
63  dst = output->data;
64  if (params->input0_broadcast || params->input1_broadcast) {
65  for (int i = 0; i < dims_count; ++i) {
66  dst[i] = pfun(params->v, src[i]);
67  }
68  } else {
69  const DnnOperand *input1 = &operands[input_operand_indexes[1]];
70  const float *src1 = input1->data;
71  for (int i = 0; i < dims_count; ++i) {
72  dst[i] = pfun(src[i], src1[i]);
73  }
74  }
75 }
76 static void math_binary_not_commutative(FunType pfun, const DnnLayerMathBinaryParams *params, const DnnOperand *input, DnnOperand *output, DnnOperand *operands, const int32_t *input_operand_indexes)
77 {
78  int dims_count;
79  const float *src;
80  float *dst;
82  src = input->data;
83  dst = output->data;
84  if (params->input0_broadcast) {
85  for (int i = 0; i < dims_count; ++i) {
86  dst[i] = pfun(params->v, src[i]);
87  }
88  } else if (params->input1_broadcast) {
89  for (int i = 0; i < dims_count; ++i) {
90  dst[i] = pfun(src[i], params->v);
91  }
92  } else {
93  const DnnOperand *input1 = &operands[input_operand_indexes[1]];
94  const float *src1 = input1->data;
95  for (int i = 0; i < dims_count; ++i) {
96  dst[i] = pfun(src[i], src1[i]);
97  }
98  }
99 }
100 int ff_dnn_load_layer_math_binary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
101 {
102  DnnLayerMathBinaryParams params = { 0 };
103  int dnn_size = 0;
104  int input_index = 0;
105 
106  params.bin_op = (int32_t)avio_rl32(model_file_context);
107  dnn_size += 4;
108 
109  params.input0_broadcast = (int32_t)avio_rl32(model_file_context);
110  dnn_size += 4;
111  if (params.input0_broadcast) {
112  params.v = av_int2float(avio_rl32(model_file_context));
113  } else {
114  layer->input_operand_indexes[input_index] = (int32_t)avio_rl32(model_file_context);
115  if (layer->input_operand_indexes[input_index] >= operands_num) {
116  return 0;
117  }
118  input_index++;
119  }
120  dnn_size += 4;
121 
122  params.input1_broadcast = (int32_t)avio_rl32(model_file_context);
123  dnn_size += 4;
124  if (params.input1_broadcast) {
125  params.v = av_int2float(avio_rl32(model_file_context));
126  } else {
127  layer->input_operand_indexes[input_index] = (int32_t)avio_rl32(model_file_context);
128  if (layer->input_operand_indexes[input_index] >= operands_num) {
129  return 0;
130  }
131  input_index++;
132  }
133  dnn_size += 4;
134 
135  layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
136  dnn_size += 4;
137 
138  if (layer->output_operand_index >= operands_num) {
139  return 0;
140  }
141  layer->params = av_memdup(&params, sizeof(params));
142  if (!layer->params)
143  return 0;
144 
145  return dnn_size;
146 }
147 
148 int ff_dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_operand_indexes,
149  int32_t output_operand_index, const void *parameters, NativeContext *ctx)
150 {
151  const DnnOperand *input = &operands[input_operand_indexes[0]];
152  DnnOperand *output = &operands[output_operand_index];
153  const DnnLayerMathBinaryParams *params = parameters;
154 
155  for (int i = 0; i < 4; ++i)
156  output->dims[i] = input->dims[i];
157 
158  output->data_type = input->data_type;
160  if (output->length <= 0) {
161  av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
162  return AVERROR(EINVAL);
163  }
164  output->data = av_realloc(output->data, output->length);
165  if (!output->data) {
166  av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
167  return AVERROR(ENOMEM);
168  }
169 
170  switch (params->bin_op) {
171  case DMBO_SUB:
172  math_binary_not_commutative(sub, params, input, output, operands, input_operand_indexes);
173  return 0;
174  case DMBO_ADD:
175  math_binary_commutative(add, params, input, output, operands, input_operand_indexes);
176  return 0;
177  case DMBO_MUL:
178  math_binary_commutative(mul, params, input, output, operands, input_operand_indexes);
179  return 0;
180  case DMBO_REALDIV:
181  math_binary_not_commutative(realdiv, params, input, output, operands, input_operand_indexes);
182  return 0;
183  case DMBO_MINIMUM:
184  math_binary_commutative(minimum, params, input, output, operands, input_operand_indexes);
185  return 0;
186  case DMBO_FLOORMOD:
187  math_binary_not_commutative(floormod, params, input, output, operands, input_operand_indexes);
188  return 0;
189  default:
190  av_log(ctx, AV_LOG_ERROR, "Unmatch math binary operator\n");
191  return AVERROR(EINVAL);
192  }
193 }
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
DnnLayerMathBinaryParams::input0_broadcast
int input0_broadcast
Definition: dnn_backend_native_layer_mathbinary.h:45
sub
static float sub(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:31
src1
const pixel * src1
Definition: h264pred_template.c:421
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
DnnLayerMathBinaryParams
Definition: dnn_backend_native_layer_mathbinary.h:43
DnnLayerMathBinaryParams::v
float v
Definition: dnn_backend_native_layer_mathbinary.h:47
DnnLayerMathBinaryParams::input1_broadcast
int input1_broadcast
Definition: dnn_backend_native_layer_mathbinary.h:46
floormod
static float floormod(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:51
av_memdup
void * av_memdup(const void *p, size_t size)
Duplicate a buffer with av_malloc().
Definition: mem.c:302
dnn_backend_native_layer_mathbinary.h
realdiv
static float realdiv(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:43
av_int2float
static av_always_inline float av_int2float(uint32_t i)
Reinterpret a 32-bit integer as a float.
Definition: intfloat.h:40
ff_calculate_operand_data_length
int32_t ff_calculate_operand_data_length(const DnnOperand *oprd)
Definition: dnn_backend_native.c:503
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
float
float
Definition: af_crystalizer.c:122
DnnOperand::data
void * data
data pointer with data length in bytes.
Definition: dnn_backend_native.h:104
ctx
AVFormatContext * ctx
Definition: movenc.c:48
math_binary_not_commutative
static void math_binary_not_commutative(FunType pfun, const DnnLayerMathBinaryParams *params, const DnnOperand *input, DnnOperand *output, DnnOperand *operands, const int32_t *input_operand_indexes)
Definition: dnn_backend_native_layer_mathbinary.c:76
mul
static float mul(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:39
Layer::params
void * params
Definition: dnn_backend_native.h:66
DMBO_MUL
@ DMBO_MUL
Definition: dnn_backend_native_layer_mathbinary.h:36
DMBO_MINIMUM
@ DMBO_MINIMUM
Definition: dnn_backend_native_layer_mathbinary.h:38
avio_rl32
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:751
minimum
static float minimum(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:47
AVIOContext
Bytestream IO Context.
Definition: avio.h:166
Layer::output_operand_index
int32_t output_operand_index
Definition: dnn_backend_native.h:65
NativeContext
Definition: dnn_backend_native.h:118
Layer
Definition: dnn_backend_native.h:57
Layer::input_operand_indexes
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
Definition: dnn_backend_native.h:64
DMBO_ADD
@ DMBO_ADD
Definition: dnn_backend_native_layer_mathbinary.h:35
DMBO_FLOORMOD
@ DMBO_FLOORMOD
Definition: dnn_backend_native_layer_mathbinary.h:39
dnn_backend_native.h
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
ff_dnn_load_layer_math_binary
int ff_dnn_load_layer_math_binary(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
Definition: dnn_backend_native_layer_mathbinary.c:100
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
FFMIN
#define FFMIN(a, b)
Definition: macros.h:49
math_binary_commutative
static void math_binary_commutative(FunType pfun, const DnnLayerMathBinaryParams *params, const DnnOperand *input, DnnOperand *output, DnnOperand *operands, const int32_t *input_operand_indexes)
Definition: dnn_backend_native_layer_mathbinary.c:56
DMBO_SUB
@ DMBO_SUB
Definition: dnn_backend_native_layer_mathbinary.h:34
DnnOperand
Definition: dnn_backend_native.h:69
ff_calculate_operand_dims_count
int32_t ff_calculate_operand_dims_count(const DnnOperand *oprd)
Definition: dnn_backend_native.c:494
DMBO_REALDIV
@ DMBO_REALDIV
Definition: dnn_backend_native_layer_mathbinary.h:37
src0
const pixel *const src0
Definition: h264pred_template.c:420
ff_dnn_execute_layer_math_binary
int ff_dnn_execute_layer_math_binary(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, const void *parameters, NativeContext *ctx)
Definition: dnn_backend_native_layer_mathbinary.c:148
add
static float add(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:35
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
int32_t
int32_t
Definition: audioconvert.c:56
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
DnnLayerMathBinaryParams::bin_op
DNNMathBinaryOperation bin_op
Definition: dnn_backend_native_layer_mathbinary.h:44
int
int
Definition: ffmpeg_filter.c:157
FunType
float(* FunType)(float src0, float src1)
Definition: dnn_backend_native_layer_mathbinary.c:29
av_realloc
void * av_realloc(void *ptr, size_t size)
Allocate, reallocate, or free a block of memory.
Definition: mem.c:153