FFmpeg
dnn_backend_torch.cpp
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2024
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 /**
22  * @file
23  * DNN Torch backend implementation.
24  */
25 
26 #include <torch/torch.h>
27 #include <torch/script.h>
28 
29 extern "C" {
30 #include "dnn_io_proc.h"
31 #include "dnn_backend_common.h"
32 #include "libavutil/opt.h"
33 #include "libavutil/mem.h"
34 #include "queue.h"
35 #include "safe_queue.h"
36 }
37 
38 typedef struct THModel {
41  torch::jit::Module *jit_model;
45 } THModel;
46 
47 typedef struct THInferRequest {
48  torch::Tensor *output;
49  torch::Tensor *input_tensor;
51 
52 typedef struct THRequestItem {
57 
58 
59 #define OFFSET(x) offsetof(THOptions, x)
60 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM
61 static const AVOption dnn_th_options[] = {
62  { "optimize", "turn on graph executor optimization", OFFSET(optimize), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, FLAGS},
63  { NULL }
64 };
65 
66 static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
67 {
68  THModel *th_model = (THModel *)task->model;
69  DnnContext *ctx = th_model->ctx;
70  LastLevelTaskItem *lltask = (LastLevelTaskItem *)av_malloc(sizeof(*lltask));
71  if (!lltask) {
72  av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for LastLevelTaskItem\n");
73  return AVERROR(ENOMEM);
74  }
75  task->inference_todo = 1;
76  task->inference_done = 0;
77  lltask->task = task;
78  if (ff_queue_push_back(lltask_queue, lltask) < 0) {
79  av_log(ctx, AV_LOG_ERROR, "Failed to push back lltask_queue.\n");
80  av_freep(&lltask);
81  return AVERROR(ENOMEM);
82  }
83  return 0;
84 }
85 
86 static void th_free_request(THInferRequest *request)
87 {
88  if (!request)
89  return;
90  if (request->output) {
91  delete(request->output);
92  request->output = NULL;
93  }
94  if (request->input_tensor) {
95  delete(request->input_tensor);
96  request->input_tensor = NULL;
97  }
98  return;
99 }
100 
102 {
103  THRequestItem *item;
104  if (!arg || !*arg) {
105  return;
106  }
107  item = *arg;
109  av_freep(&item->infer_request);
110  av_freep(&item->lltask);
112  av_freep(arg);
113 }
114 
115 static void dnn_free_model_th(DNNModel **model)
116 {
117  THModel *th_model;
118  if (!model || !*model)
119  return;
120 
121  th_model = (THModel *) (*model);
122  while (ff_safe_queue_size(th_model->request_queue) != 0) {
124  destroy_request_item(&item);
125  }
127 
128  while (ff_queue_size(th_model->lltask_queue) != 0) {
130  av_freep(&item);
131  }
132  ff_queue_destroy(th_model->lltask_queue);
133 
134  while (ff_queue_size(th_model->task_queue) != 0) {
135  TaskItem *item = (TaskItem *)ff_queue_pop_front(th_model->task_queue);
136  av_frame_free(&item->in_frame);
137  av_frame_free(&item->out_frame);
138  av_freep(&item);
139  }
140  ff_queue_destroy(th_model->task_queue);
141  delete th_model->jit_model;
142  av_freep(&th_model);
143  *model = NULL;
144 }
145 
146 static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
147 {
148  input->dt = DNN_FLOAT;
149  input->order = DCO_RGB;
150  input->layout = DL_NCHW;
151  input->dims[0] = 1;
152  input->dims[1] = 3;
153  input->dims[2] = -1;
154  input->dims[3] = -1;
155  return 0;
156 }
157 
158 static void deleter(void *arg)
159 {
160  av_freep(&arg);
161 }
162 
163 static int fill_model_input_th(THModel *th_model, THRequestItem *request)
164 {
165  LastLevelTaskItem *lltask = NULL;
166  TaskItem *task = NULL;
167  THInferRequest *infer_request = NULL;
168  DNNData input = { 0 };
169  DnnContext *ctx = th_model->ctx;
170  int ret, width_idx, height_idx, channel_idx;
171 
172  lltask = (LastLevelTaskItem *)ff_queue_pop_front(th_model->lltask_queue);
173  if (!lltask) {
174  ret = AVERROR(EINVAL);
175  goto err;
176  }
177  request->lltask = lltask;
178  task = lltask->task;
179  infer_request = request->infer_request;
180 
181  ret = get_input_th(&th_model->model, &input, NULL);
182  if ( ret != 0) {
183  goto err;
184  }
185  width_idx = dnn_get_width_idx_by_layout(input.layout);
186  height_idx = dnn_get_height_idx_by_layout(input.layout);
187  channel_idx = dnn_get_channel_idx_by_layout(input.layout);
188  input.dims[height_idx] = task->in_frame->height;
189  input.dims[width_idx] = task->in_frame->width;
190  input.data = av_malloc(input.dims[height_idx] * input.dims[width_idx] *
191  input.dims[channel_idx] * sizeof(float));
192  if (!input.data)
193  return AVERROR(ENOMEM);
194  infer_request->input_tensor = new torch::Tensor();
195  infer_request->output = new torch::Tensor();
196 
197  switch (th_model->model.func_type) {
198  case DFT_PROCESS_FRAME:
199  input.scale = 255;
200  if (task->do_ioproc) {
201  if (th_model->model.frame_pre_proc != NULL) {
202  th_model->model.frame_pre_proc(task->in_frame, &input, th_model->model.filter_ctx);
203  } else {
205  }
206  }
207  break;
208  default:
209  avpriv_report_missing_feature(NULL, "model function type %d", th_model->model.func_type);
210  break;
211  }
212  *infer_request->input_tensor = torch::from_blob(input.data,
213  {1, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]},
214  deleter, torch::kFloat32);
215  return 0;
216 
217 err:
218  th_free_request(infer_request);
219  return ret;
220 }
221 
222 static int th_start_inference(void *args)
223 {
224  THRequestItem *request = (THRequestItem *)args;
225  THInferRequest *infer_request = NULL;
226  LastLevelTaskItem *lltask = NULL;
227  TaskItem *task = NULL;
228  THModel *th_model = NULL;
229  DnnContext *ctx = NULL;
230  std::vector<torch::jit::IValue> inputs;
231  torch::NoGradGuard no_grad;
232 
233  if (!request) {
234  av_log(NULL, AV_LOG_ERROR, "THRequestItem is NULL\n");
235  return AVERROR(EINVAL);
236  }
237  infer_request = request->infer_request;
238  lltask = request->lltask;
239  task = lltask->task;
240  th_model = (THModel *)task->model;
241  ctx = th_model->ctx;
242 
243  if (ctx->torch_option.optimize)
244  torch::jit::setGraphExecutorOptimize(true);
245  else
246  torch::jit::setGraphExecutorOptimize(false);
247 
248  if (!infer_request->input_tensor || !infer_request->output) {
249  av_log(ctx, AV_LOG_ERROR, "input or output tensor is NULL\n");
250  return DNN_GENERIC_ERROR;
251  }
252  // Transfer tensor to the same device as model
253  c10::Device device = (*th_model->jit_model->parameters().begin()).device();
254  if (infer_request->input_tensor->device() != device)
255  *infer_request->input_tensor = infer_request->input_tensor->to(device);
256  inputs.push_back(*infer_request->input_tensor);
257 
258  *infer_request->output = th_model->jit_model->forward(inputs).toTensor();
259 
260  return 0;
261 }
262 
263 static void infer_completion_callback(void *args) {
264  THRequestItem *request = (THRequestItem*)args;
265  LastLevelTaskItem *lltask = request->lltask;
266  TaskItem *task = lltask->task;
267  DNNData outputs = { 0 };
268  THInferRequest *infer_request = request->infer_request;
269  THModel *th_model = (THModel *)task->model;
270  torch::Tensor *output = infer_request->output;
271 
272  c10::IntArrayRef sizes = output->sizes();
273  outputs.order = DCO_RGB;
274  outputs.layout = DL_NCHW;
275  outputs.dt = DNN_FLOAT;
276  if (sizes.size() == 4) {
277  // 4 dimensions: [batch_size, channel, height, width]
278  // this format of data is normally used for video frame SR
279  outputs.dims[0] = sizes.at(0); // N
280  outputs.dims[1] = sizes.at(1); // C
281  outputs.dims[2] = sizes.at(2); // H
282  outputs.dims[3] = sizes.at(3); // W
283  } else {
284  avpriv_report_missing_feature(th_model->ctx, "Support of this kind of model");
285  goto err;
286  }
287 
288  switch (th_model->model.func_type) {
289  case DFT_PROCESS_FRAME:
290  if (task->do_ioproc) {
291  // Post process can only deal with CPU memory.
292  if (output->device() != torch::kCPU)
293  *output = output->to(torch::kCPU);
294  outputs.scale = 255;
295  outputs.data = output->data_ptr();
296  if (th_model->model.frame_post_proc != NULL) {
297  th_model->model.frame_post_proc(task->out_frame, &outputs, th_model->model.filter_ctx);
298  } else {
299  ff_proc_from_dnn_to_frame(task->out_frame, &outputs, th_model->ctx);
300  }
301  } else {
304  }
305  break;
306  default:
307  avpriv_report_missing_feature(th_model->ctx, "model function type %d", th_model->model.func_type);
308  goto err;
309  }
310  task->inference_done++;
311  av_freep(&request->lltask);
312 err:
313  th_free_request(infer_request);
314 
315  if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
316  destroy_request_item(&request);
317  av_log(th_model->ctx, AV_LOG_ERROR, "Unable to push back request_queue when failed to start inference.\n");
318  }
319 }
320 
321 static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
322 {
323  THModel *th_model = NULL;
324  LastLevelTaskItem *lltask;
325  TaskItem *task = NULL;
326  int ret = 0;
327 
328  if (ff_queue_size(lltask_queue) == 0) {
329  destroy_request_item(&request);
330  return 0;
331  }
332 
333  lltask = (LastLevelTaskItem *)ff_queue_peek_front(lltask_queue);
334  if (lltask == NULL) {
335  av_log(NULL, AV_LOG_ERROR, "Failed to get LastLevelTaskItem\n");
336  ret = AVERROR(EINVAL);
337  goto err;
338  }
339  task = lltask->task;
340  th_model = (THModel *)task->model;
341 
342  ret = fill_model_input_th(th_model, request);
343  if ( ret != 0) {
344  goto err;
345  }
346  if (task->async) {
347  avpriv_report_missing_feature(th_model->ctx, "LibTorch async");
348  } else {
349  ret = th_start_inference((void *)(request));
350  if (ret != 0) {
351  goto err;
352  }
353  infer_completion_callback(request);
354  return (task->inference_done == task->inference_todo) ? 0 : DNN_GENERIC_ERROR;
355  }
356 
357 err:
358  th_free_request(request->infer_request);
359  if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) {
360  destroy_request_item(&request);
361  }
362  return ret;
363 }
364 
365 static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height,
366  const char *output_name, int *output_width, int *output_height)
367 {
368  int ret = 0;
369  THModel *th_model = (THModel*) model;
370  DnnContext *ctx = th_model->ctx;
371  TaskItem task = { 0 };
372  THRequestItem *request = NULL;
373  DNNExecBaseParams exec_params = {
374  .input_name = input_name,
375  .output_names = &output_name,
376  .nb_output = 1,
377  .in_frame = NULL,
378  .out_frame = NULL,
379  };
380  ret = ff_dnn_fill_gettingoutput_task(&task, &exec_params, th_model, input_height, input_width, ctx);
381  if ( ret != 0) {
382  goto err;
383  }
384 
385  ret = extract_lltask_from_task(&task, th_model->lltask_queue);
386  if ( ret != 0) {
387  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
388  goto err;
389  }
390 
391  request = (THRequestItem*) ff_safe_queue_pop_front(th_model->request_queue);
392  if (!request) {
393  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
394  ret = AVERROR(EINVAL);
395  goto err;
396  }
397 
398  ret = execute_model_th(request, th_model->lltask_queue);
399  *output_width = task.out_frame->width;
400  *output_height = task.out_frame->height;
401 
402 err:
403  av_frame_free(&task.out_frame);
404  av_frame_free(&task.in_frame);
405  return ret;
406 }
407 
409 {
410  THInferRequest *request = (THInferRequest *)av_malloc(sizeof(THInferRequest));
411  if (!request) {
412  return NULL;
413  }
414  request->input_tensor = NULL;
415  request->output = NULL;
416  return request;
417 }
418 
420 {
421  DNNModel *model = NULL;
422  THModel *th_model = NULL;
423  THRequestItem *item = NULL;
424  const char *device_name = ctx->device ? ctx->device : "cpu";
425 
426  th_model = (THModel *)av_mallocz(sizeof(THModel));
427  if (!th_model)
428  return NULL;
429  model = &th_model->model;
430  th_model->ctx = ctx;
431 
432  c10::Device device = c10::Device(device_name);
433  if (device.is_xpu()) {
434  if (!at::hasXPU()) {
435  av_log(ctx, AV_LOG_ERROR, "No XPU device found\n");
436  goto fail;
437  }
438  at::detail::getXPUHooks().initXPU();
439  } else if (!device.is_cpu()) {
440  av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name);
441  goto fail;
442  }
443 
444  try {
445  th_model->jit_model = new torch::jit::Module;
446  (*th_model->jit_model) = torch::jit::load(ctx->model_filename);
447  th_model->jit_model->to(device);
448  } catch (const c10::Error& e) {
449  av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n");
450  goto fail;
451  }
452 
453  th_model->request_queue = ff_safe_queue_create();
454  if (!th_model->request_queue) {
455  goto fail;
456  }
457 
458  item = (THRequestItem *)av_mallocz(sizeof(THRequestItem));
459  if (!item) {
460  goto fail;
461  }
462  item->lltask = NULL;
464  if (!item->infer_request) {
465  av_log(NULL, AV_LOG_ERROR, "Failed to allocate memory for Torch inference request\n");
466  goto fail;
467  }
470  item->exec_module.args = item;
471 
472  if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) {
473  goto fail;
474  }
475  item = NULL;
476 
477  th_model->task_queue = ff_queue_create();
478  if (!th_model->task_queue) {
479  goto fail;
480  }
481 
482  th_model->lltask_queue = ff_queue_create();
483  if (!th_model->lltask_queue) {
484  goto fail;
485  }
486 
487  model->get_input = &get_input_th;
488  model->get_output = &get_output_th;
489  model->filter_ctx = filter_ctx;
490  model->func_type = func_type;
491  return model;
492 
493 fail:
494  if (item) {
495  destroy_request_item(&item);
496  av_freep(&item);
497  }
498  dnn_free_model_th(&model);
499  return NULL;
500 }
501 
502 static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
503 {
504  THModel *th_model = (THModel *)model;
505  DnnContext *ctx = th_model->ctx;
506  TaskItem *task;
507  THRequestItem *request;
508  int ret = 0;
509 
510  ret = ff_check_exec_params(ctx, DNN_TH, model->func_type, exec_params);
511  if (ret != 0) {
512  av_log(ctx, AV_LOG_ERROR, "exec parameter checking fail.\n");
513  return ret;
514  }
515 
516  task = (TaskItem *)av_malloc(sizeof(TaskItem));
517  if (!task) {
518  av_log(ctx, AV_LOG_ERROR, "unable to alloc memory for task item.\n");
519  return AVERROR(ENOMEM);
520  }
521 
522  ret = ff_dnn_fill_task(task, exec_params, th_model, 0, 1);
523  if (ret != 0) {
524  av_freep(&task);
525  av_log(ctx, AV_LOG_ERROR, "unable to fill task.\n");
526  return ret;
527  }
528 
529  ret = ff_queue_push_back(th_model->task_queue, task);
530  if (ret < 0) {
531  av_freep(&task);
532  av_log(ctx, AV_LOG_ERROR, "unable to push back task_queue.\n");
533  return ret;
534  }
535 
536  ret = extract_lltask_from_task(task, th_model->lltask_queue);
537  if (ret != 0) {
538  av_log(ctx, AV_LOG_ERROR, "unable to extract last level task from task.\n");
539  return ret;
540  }
541 
542  request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);
543  if (!request) {
544  av_log(ctx, AV_LOG_ERROR, "unable to get infer request.\n");
545  return AVERROR(EINVAL);
546  }
547 
548  return execute_model_th(request, th_model->lltask_queue);
549 }
550 
552 {
553  THModel *th_model = (THModel *)model;
554  return ff_dnn_get_result_common(th_model->task_queue, in, out);
555 }
556 
557 static int dnn_flush_th(const DNNModel *model)
558 {
559  THModel *th_model = (THModel *)model;
560  THRequestItem *request;
561 
562  if (ff_queue_size(th_model->lltask_queue) == 0)
563  // no pending task need to flush
564  return 0;
565 
566  request = (THRequestItem *)ff_safe_queue_pop_front(th_model->request_queue);
567  if (!request) {
568  av_log(th_model->ctx, AV_LOG_ERROR, "unable to get infer request.\n");
569  return AVERROR(EINVAL);
570  }
571 
572  return execute_model_th(request, th_model->lltask_queue);
573 }
574 
575 extern const DNNModule ff_dnn_backend_torch = {
576  .clazz = DNN_DEFINE_CLASS(dnn_th),
577  .type = DNN_TH,
578  .load_model = dnn_load_model_th,
579  .execute_model = dnn_execute_model_th,
580  .get_result = dnn_get_result_th,
581  .flush = dnn_flush_th,
582  .free_model = dnn_free_model_th,
583 };
THRequestItem::lltask
LastLevelTaskItem * lltask
Definition: dnn_backend_torch.cpp:54
THModel::lltask_queue
Queue * lltask_queue
Definition: dnn_backend_torch.cpp:44
THRequestItem::infer_request
THInferRequest * infer_request
Definition: dnn_backend_torch.cpp:53
THModel::ctx
DnnContext * ctx
Definition: dnn_backend_torch.cpp:40
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
ff_safe_queue_pop_front
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
Definition: safe_queue.c:105
out
FILE * out
Definition: movenc.c:55
deleter
static void deleter(void *arg)
Definition: dnn_backend_torch.cpp:158
FLAGS
#define FLAGS
Definition: dnn_backend_torch.cpp:60
THModel
Definition: dnn_backend_torch.cpp:38
DNNAsyncExecModule
Common Async Execution Mechanism for the DNN Backends.
Definition: dnn_backend_common.h:65
DNNFunctionType
DNNFunctionType
Definition: dnn_interface.h:56
output
filter_frame For filters that do not use the this method is called when a frame is pushed to the filter s input It can be called at any time except in a reentrant way If the input frame is enough to produce output
Definition: filter_design.txt:225
ff_queue_pop_front
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
Definition: queue.c:151
ff_check_exec_params
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
Definition: dnn_backend_common.c:30
ff_queue_size
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Definition: queue.c:88
DNN_GENERIC_ERROR
#define DNN_GENERIC_ERROR
Definition: dnn_interface.h:33
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:162
LastLevelTaskItem
Definition: dnn_backend_common.h:57
ff_dnn_backend_torch
const DNNModule ff_dnn_backend_torch
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:389
AVFrame::width
int width
Definition: frame.h:461
SafeQueue
Double-ended queue with mutex locks ensuring data consistency while multithreading.
Definition: safe_queue.c:46
dnn_execute_model_th
static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
Definition: dnn_backend_torch.cpp:502
AVOption
AVOption.
Definition: opt.h:429
DNNModel::frame_pre_proc
FramePrePostProc frame_pre_proc
Definition: dnn_interface.h:110
DNNExecBaseParams::input_name
const char * input_name
Definition: dnn_interface.h:81
dnn_io_proc.h
TaskItem
Definition: dnn_backend_common.h:43
DNNAsyncExecModule::callback
void(* callback)(void *args)
Completion Callback for the backend.
Definition: dnn_backend_common.h:77
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:30
DNNModel::filter_ctx
AVFilterContext * filter_ctx
Definition: dnn_interface.h:99
ff_queue_create
Queue * ff_queue_create(void)
Create a Queue instance.
Definition: queue.c:47
dnn_get_width_idx_by_layout
static int dnn_get_width_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:197
TaskItem::model
void * model
Definition: dnn_backend_common.h:44
fail
#define fail()
Definition: checkasm.h:188
DnnContext
Definition: dnn_interface.h:143
filter_ctx
static FilteringContext * filter_ctx
Definition: transcode.c:52
Queue
Linear double-ended data structure.
Definition: queue.c:33
ff_queue_push_back
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
Definition: queue.c:130
THModel::jit_model
torch::jit::Module * jit_model
Definition: dnn_backend_torch.cpp:41
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:209
LastLevelTaskItem::task
TaskItem * task
Definition: dnn_backend_common.h:58
destroy_request_item
static void destroy_request_item(THRequestItem **arg)
Definition: dnn_backend_torch.cpp:101
th_create_inference_request
static THInferRequest * th_create_inference_request(void)
Definition: dnn_backend_torch.cpp:408
ff_queue_destroy
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
Definition: queue.c:72
DNNData
Definition: dnn_interface.h:69
DNNModule::clazz
const AVClass clazz
Definition: dnn_interface.h:176
ff_dnn_fill_gettingoutput_task
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
Definition: dnn_backend_common.c:156
DNNModel::get_output
int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_interface.h:106
ctx
AVFormatContext * ctx
Definition: movenc.c:49
TaskItem::inference_todo
uint32_t inference_todo
Definition: dnn_backend_common.h:52
DL_NCHW
@ DL_NCHW
Definition: dnn_interface.h:65
dnn_load_model_th
static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
Definition: dnn_backend_torch.cpp:419
arg
const char * arg
Definition: jacosubdec.c:67
if
if(ret)
Definition: filter_design.txt:179
ff_safe_queue_size
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
Definition: safe_queue.c:80
ff_proc_from_frame_to_dnn
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
Definition: dnn_io_proc.c:182
THRequestItem::exec_module
DNNAsyncExecModule exec_module
Definition: dnn_backend_torch.cpp:55
NULL
#define NULL
Definition: coverity.c:32
sizes
static const int sizes[][2]
Definition: img2dec.c:60
get_input_th
static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
Definition: dnn_backend_torch.cpp:146
ff_safe_queue_create
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
Definition: safe_queue.c:52
DNNModel::frame_post_proc
FramePrePostProc frame_post_proc
Definition: dnn_interface.h:113
get_output_th
static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
Definition: dnn_backend_torch.cpp:365
ff_dnn_async_module_cleanup
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
Definition: dnn_backend_common.c:86
infer_completion_callback
static void infer_completion_callback(void *args)
Definition: dnn_backend_torch.cpp:263
TaskItem::in_frame
AVFrame * in_frame
Definition: dnn_backend_common.h:45
extract_lltask_from_task
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
Definition: dnn_backend_torch.cpp:66
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
THInferRequest::output
torch::Tensor * output
Definition: dnn_backend_torch.cpp:48
TaskItem::async
uint8_t async
Definition: dnn_backend_common.h:49
TaskItem::inference_done
uint32_t inference_done
Definition: dnn_backend_common.h:53
queue.h
DNNModel::func_type
DNNFunctionType func_type
Definition: dnn_interface.h:101
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
ff_safe_queue_destroy
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
Definition: safe_queue.c:69
DNN_FLOAT
@ DNN_FLOAT
Definition: dnn_interface.h:41
dnn_get_result_th
static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)
Definition: dnn_backend_torch.cpp:551
ff_dnn_fill_task
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
Definition: dnn_backend_common.c:50
input
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
Definition: filter_design.txt:172
DNN_DEFINE_CLASS
#define DNN_DEFINE_CLASS(fname)
Definition: dnn_backend_common.h:39
THRequestItem
Definition: dnn_backend_torch.cpp:52
ff_safe_queue_push_back
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
Definition: safe_queue.c:95
th_start_inference
static int th_start_inference(void *args)
Definition: dnn_backend_torch.cpp:222
THInferRequest::input_tensor
torch::Tensor * input_tensor
Definition: dnn_backend_torch.cpp:49
DNNAsyncExecModule::start_inference
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
Definition: dnn_backend_common.h:70
DNNAsyncExecModule::args
void * args
Argument for the execution functions.
Definition: dnn_backend_common.h:83
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:256
safe_queue.h
THInferRequest
Definition: dnn_backend_torch.cpp:47
outputs
static const AVFilterPad outputs[]
Definition: af_aap.c:310
ret
ret
Definition: filter_design.txt:187
TaskItem::out_frame
AVFrame * out_frame
Definition: dnn_backend_common.h:46
AVFrame::height
int height
Definition: frame.h:461
dnn_backend_common.h
THModel::model
DNNModel model
Definition: dnn_backend_torch.cpp:39
dnn_th_options
static const AVOption dnn_th_options[]
Definition: dnn_backend_torch.cpp:61
execute_model_th
static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
Definition: dnn_backend_torch.cpp:321
OFFSET
#define OFFSET(x)
Definition: dnn_backend_torch.cpp:59
AV_OPT_TYPE_INT
@ AV_OPT_TYPE_INT
Underlying C type is int.
Definition: opt.h:259
ff_dnn_get_result_common
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
Definition: dnn_backend_common.c:136
ff_queue_peek_front
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
Definition: queue.c:93
DCO_RGB
@ DCO_RGB
Definition: dnn_interface.h:46
AVFilterContext
An instance of a filter.
Definition: avfilter.h:457
DNNModel
Definition: dnn_interface.h:97
DNN_TH
@ DNN_TH
Definition: dnn_interface.h:38
mem.h
dnn_get_height_idx_by_layout
static int dnn_get_height_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:202
dnn_flush_th
static int dnn_flush_th(const DNNModel *model)
Definition: dnn_backend_torch.cpp:557
THModel::task_queue
Queue * task_queue
Definition: dnn_backend_torch.cpp:43
dnn_get_channel_idx_by_layout
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
Definition: dnn_interface.h:207
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
DNNExecBaseParams
Definition: dnn_interface.h:80
DNNModel::get_input
int(* get_input)(struct DNNModel *model, DNNData *input, const char *input_name)
Definition: dnn_interface.h:104
dnn_free_model_th
static void dnn_free_model_th(DNNModel **model)
Definition: dnn_backend_torch.cpp:115
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
TaskItem::do_ioproc
uint8_t do_ioproc
Definition: dnn_backend_common.h:50
DNNAsyncStatusType
DNNAsyncStatusType
Definition: dnn_interface.h:49
DFT_PROCESS_FRAME
@ DFT_PROCESS_FRAME
Definition: dnn_interface.h:58
DNNModule
Definition: dnn_interface.h:175
fill_model_input_th
static int fill_model_input_th(THModel *th_model, THRequestItem *request)
Definition: dnn_backend_torch.cpp:163
THModel::request_queue
SafeQueue * request_queue
Definition: dnn_backend_torch.cpp:42
ff_proc_from_dnn_to_frame
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
Definition: dnn_io_proc.c:42
th_free_request
static void th_free_request(THInferRequest *request)
Definition: dnn_backend_torch.cpp:86