FFmpeg
dnn_backend_native_layer_pad.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Guo Yejun
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <string.h>
22 #include "libavutil/avassert.h"
24 
25 int ff_dnn_load_layer_pad(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
26 {
27  LayerPadParams *params;
28  int dnn_size = 0;
29  params = av_malloc(sizeof(*params));
30  if (!params)
31  return 0;
32 
33  params->mode = (int32_t)avio_rl32(model_file_context);
34  dnn_size += 4;
35  for (int i = 0; i < 4; ++i) {
36  params->paddings[i][0] = avio_rl32(model_file_context);
37  params->paddings[i][1] = avio_rl32(model_file_context);
38  dnn_size += 8;
39  }
40  layer->input_operand_indexes[0] = (int32_t)avio_rl32(model_file_context);
41  layer->output_operand_index = (int32_t)avio_rl32(model_file_context);
42  dnn_size += 8;
43  layer->params = params;
44 
45  if (layer->input_operand_indexes[0] >= operands_num || layer->output_operand_index >= operands_num) {
46  return 0;
47  }
48 
49  return dnn_size;
50 }
51 
52 static int before_get_buddy(int given, int paddings, LayerPadModeParam mode)
53 {
54  if (mode == LPMP_SYMMETRIC) {
55  return (2 * paddings - 1 - given);
56  } else if (mode == LPMP_REFLECT) {
57  return (2 * paddings - given);
58  } else {
59  av_assert0(!"should not reach here");
60  return 0;
61  }
62 }
63 
64 static int after_get_buddy(int given, int border, LayerPadModeParam mode)
65 {
66  if (mode == LPMP_SYMMETRIC) {
67  int offset = given - border;
68  return (border - 1 - offset);
69  } else if (mode == LPMP_REFLECT) {
70  int offset = given - border;
71  return (border - 2 - offset);
72  } else {
73  av_assert0(!"should not reach here");
74  return 0;
75  }
76 }
77 
78 int ff_dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes,
79  int32_t output_operand_index, const void *parameters, NativeContext *ctx)
80 {
81  int32_t before_paddings;
82  int32_t after_paddings;
83  float* output;
84  const LayerPadParams *params = parameters;
85 
86  // suppose format is <N, H, W, C>
87  int32_t input_operand_index = input_operand_indexes[0];
88  int number = operands[input_operand_index].dims[0];
89  int height = operands[input_operand_index].dims[1];
90  int width = operands[input_operand_index].dims[2];
91  int channel = operands[input_operand_index].dims[3];
92  const float *input = operands[input_operand_index].data;
93 
94  int new_number = number + params->paddings[0][0] + params->paddings[0][1];
95  int new_height = height + params->paddings[1][0] + params->paddings[1][1];
96  int new_width = width + params->paddings[2][0] + params->paddings[2][1];
97  int new_channel = channel + params->paddings[3][0] + params->paddings[3][1];
98 
99  int c_stride = channel;
100  int wc_stride = c_stride * width;
101  int hwc_stride = wc_stride * height;
102 
103  int new_c_stride = new_channel;
104  int new_wc_stride = new_c_stride * new_width;
105  int new_hwc_stride = new_wc_stride * new_height;
106 
107  DnnOperand *output_operand = &operands[output_operand_index];
108  output_operand->dims[0] = new_number;
109  output_operand->dims[1] = new_height;
110  output_operand->dims[2] = new_width;
111  output_operand->dims[3] = new_channel;
112  output_operand->data_type = operands[input_operand_index].data_type;
113  output_operand->length = ff_calculate_operand_data_length(output_operand);
114  if (output_operand->length <= 0) {
115  av_log(ctx, AV_LOG_ERROR, "The output data length overflow\n");
116  return AVERROR(EINVAL);
117  }
118  output_operand->data = av_realloc(output_operand->data, output_operand->length);
119  if (!output_operand->data) {
120  av_log(ctx, AV_LOG_ERROR, "Failed to reallocate memory for output\n");
121  return AVERROR(ENOMEM);
122  }
123  output = output_operand->data;
124 
125  // copy the original data
126  for (int n = 0; n < number; n++) {
127  for (int h = 0; h < height; h++) {
128  for (int w = 0; w < width; w++) {
129  const float *src = input + n * hwc_stride + h * wc_stride + w * c_stride;
130  float *dst = output + (n + params->paddings[0][0]) * new_hwc_stride
131  + (h + params->paddings[1][0]) * new_wc_stride
132  + (w + params->paddings[2][0]) * new_c_stride
133  + params->paddings[3][0];
134  memcpy(dst, src, channel * sizeof(float));
135  }
136  }
137  }
138 
139  // handle the first dimension
140  before_paddings = params->paddings[0][0];
141  after_paddings = params->paddings[0][1];
142  for (int n = 0; n < before_paddings; n++) {
143  float *dst = output + n * new_hwc_stride;
144  if (params->mode == LPMP_CONSTANT) {
145  for (int i = 0; i < new_hwc_stride; i++) {
146  dst[i] = params->constant_values;
147  }
148  }
149  else {
150  int buddy = before_get_buddy(n, before_paddings, params->mode);
151  float *src = output + buddy * new_hwc_stride;
152  memcpy(dst, src, new_hwc_stride * sizeof(float));
153  }
154  }
155  for (int n = 0; n < after_paddings; n++) {
156  int given = number + before_paddings + n;
157  float *dst = output + given * new_hwc_stride;
158  if (params->mode == LPMP_CONSTANT) {
159  for (int i = 0; i < new_hwc_stride; i++) {
160  dst[i] = params->constant_values;
161  }
162  } else {
163  int buddy = after_get_buddy(given, number + before_paddings, params->mode);
164  float *src = output + buddy * new_hwc_stride;
165  memcpy(dst, src, new_hwc_stride * sizeof(float));
166  }
167  }
168 
169  // handle the second dimension
170  before_paddings = params->paddings[1][0];
171  after_paddings = params->paddings[1][1];
172  for (int n = 0; n < new_number; n++) {
173  float *start = output + n * new_hwc_stride;
174  for (int h = 0; h < before_paddings; h++) {
175  float *dst = start + h * new_wc_stride;
176  if (params->mode == LPMP_CONSTANT) {
177  for (int i = 0; i < new_wc_stride; i++) {
178  dst[i] = params->constant_values;
179  }
180  } else {
181  int buddy = before_get_buddy(h, before_paddings, params->mode);
182  float *src = start + buddy * new_wc_stride;
183  memcpy(dst, src, new_wc_stride * sizeof(float));
184  }
185  }
186  for (int h = 0; h < after_paddings; h++) {
187  int given = height + before_paddings + h;
188  float *dst = start + given * new_wc_stride;
189  if (params->mode == LPMP_CONSTANT) {
190  for (int i = 0; i < new_wc_stride; i++) {
191  dst[i] = params->constant_values;
192  }
193  } else {
194  int buddy = after_get_buddy(given, height + before_paddings, params->mode);
195  float *src = start + buddy * new_wc_stride;
196  memcpy(dst, src, new_wc_stride * sizeof(float));
197  }
198  }
199  }
200 
201  // handle the third dimension
202  before_paddings = params->paddings[2][0];
203  after_paddings = params->paddings[2][1];
204  for (int n = 0; n < new_number; n++) {
205  for (int h = 0; h < new_height; h++) {
206  float *start = output + n * new_hwc_stride + h * new_wc_stride;
207  for (int w = 0; w < before_paddings; w++) {
208  float *dst = start + w * new_c_stride;
209  if (params->mode == LPMP_CONSTANT) {
210  for (int i = 0; i < new_c_stride; i++) {
211  dst[i] = params->constant_values;
212  }
213  } else {
214  int buddy = before_get_buddy(w, before_paddings, params->mode);
215  float *src = start + buddy * new_c_stride;
216  memcpy(dst, src, new_c_stride * sizeof(float));
217  }
218  }
219  for (int w = 0; w < after_paddings; w++) {
220  int given = width + before_paddings + w;
221  float *dst = start + given * new_c_stride;
222  if (params->mode == LPMP_CONSTANT) {
223  for (int i = 0; i < new_c_stride; i++) {
224  dst[i] = params->constant_values;
225  }
226  } else {
227  int buddy = after_get_buddy(given, width + before_paddings, params->mode);
228  float *src = start + buddy * new_c_stride;
229  memcpy(dst, src, new_c_stride * sizeof(float));
230  }
231  }
232  }
233  }
234 
235  // handle the fourth dimension
236  before_paddings = params->paddings[3][0];
237  after_paddings = params->paddings[3][1];
238  for (int n = 0; n < new_number; n++) {
239  for (int h = 0; h < new_height; h++) {
240  for (int w = 0; w < new_width; w++) {
241  float *start = output + n * new_hwc_stride + h * new_wc_stride + w * new_c_stride;
242  for (int c = 0; c < before_paddings; c++) {
243  float *dst = start + c;
244  if (params->mode == LPMP_CONSTANT) {
245  *dst = params->constant_values;
246  } else {
247  int buddy = before_get_buddy(c, before_paddings, params->mode);
248  float *src = start + buddy;
249  *dst = *src;
250  }
251  }
252  for (int c = 0; c < after_paddings; c++) {
253  int given = channel + before_paddings + c;
254  float *dst = start + given;
255  if (params->mode == LPMP_CONSTANT) {
256  *dst = params->constant_values;
257  } else {
258  int buddy = after_get_buddy(given, channel + before_paddings, params->mode);
259  float *src = start + buddy;
260  *dst = *src;
261  }
262  }
263  }
264  }
265  }
266 
267  return 0;
268 }
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
before_get_buddy
static int before_get_buddy(int given, int paddings, LayerPadModeParam mode)
Definition: dnn_backend_native_layer_pad.c:52
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
w
uint8_t w
Definition: llviddspenc.c:38
LayerPadModeParam
LayerPadModeParam
Definition: dnn_backend_native_layer_pad.h:31
LPMP_REFLECT
@ LPMP_REFLECT
Definition: dnn_backend_native_layer_pad.h:31
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
dnn_backend_native_layer_pad.h
ff_calculate_operand_data_length
int32_t ff_calculate_operand_data_length(const DnnOperand *oprd)
Definition: dnn_backend_native.c:503
ff_dnn_execute_layer_pad
int ff_dnn_execute_layer_pad(DnnOperand *operands, const int32_t *input_operand_indexes, int32_t output_operand_index, const void *parameters, NativeContext *ctx)
Definition: dnn_backend_native_layer_pad.c:78
avassert.h
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
LayerPadParams::mode
LayerPadModeParam mode
Definition: dnn_backend_native_layer_pad.h:35
width
#define width
DnnOperand::data
void * data
data pointer with data length in bytes.
Definition: dnn_backend_native.h:104
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
DnnOperand::data_type
DNNDataType data_type
support different kinds of data type such as float, half float, int8 etc, first support float now.
Definition: dnn_backend_native.h:85
LayerPadParams
Definition: dnn_backend_native_layer_pad.h:33
ctx
AVFormatContext * ctx
Definition: movenc.c:48
Layer::params
void * params
Definition: dnn_backend_native.h:66
DnnOperand::dims
int32_t dims[4]
there are two memory layouts, NHWC or NCHW, so we use dims, dims[0] is Number.
Definition: dnn_backend_native.h:74
c
Undefined Behavior In the C some operations are like signed integer dereferencing freed accessing outside allocated Undefined Behavior must not occur in a C it is not safe even if the output of undefined operations is unused The unsafety may seem nit picking but Optimizing compilers have in fact optimized code on the assumption that no undefined Behavior occurs Optimizing code based on wrong assumptions can and has in some cases lead to effects beyond the output of computations The signed integer overflow problem in speed critical code Code which is highly optimized and works with signed integers sometimes has the problem that often the output of the computation does not c
Definition: undefined.txt:32
LPMP_SYMMETRIC
@ LPMP_SYMMETRIC
Definition: dnn_backend_native_layer_pad.h:31
DnnOperand::length
int32_t length
Definition: dnn_backend_native.h:105
avio_rl32
unsigned int avio_rl32(AVIOContext *s)
Definition: aviobuf.c:751
AVIOContext
Bytestream IO Context.
Definition: avio.h:166
Layer::output_operand_index
int32_t output_operand_index
Definition: dnn_backend_native.h:65
LPMP_CONSTANT
@ LPMP_CONSTANT
Definition: dnn_backend_native_layer_pad.h:31
NativeContext
Definition: dnn_backend_native.h:118
Layer
Definition: dnn_backend_native.h:57
Layer::input_operand_indexes
int32_t input_operand_indexes[4]
a layer can have multiple inputs and one output.
Definition: dnn_backend_native.h:64
height
#define height
offset
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
Definition: writing_filters.txt:86
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
ff_dnn_load_layer_pad
int ff_dnn_load_layer_pad(Layer *layer, AVIOContext *model_file_context, int file_size, int operands_num)
Definition: dnn_backend_native_layer_pad.c:25
LayerPadParams::paddings
int32_t paddings[4][2]
Definition: dnn_backend_native_layer_pad.h:34
DnnOperand
Definition: dnn_backend_native.h:69
mode
mode
Definition: ebur128.h:83
after_get_buddy
static int after_get_buddy(int given, int border, LayerPadModeParam mode)
Definition: dnn_backend_native_layer_pad.c:64
LayerPadParams::constant_values
float constant_values
Definition: dnn_backend_native_layer_pad.h:36
src
INIT_CLIP pixel * src
Definition: h264pred_template.c:418
int32_t
int32_t
Definition: audioconvert.c:56
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
h
h
Definition: vp9dsp_template.c:2038
channel
channel
Definition: ebur128.h:39
av_realloc
void * av_realloc(void *ptr, size_t size)
Allocate, reallocate, or free a block of memory.
Definition: mem.c:153