26#include <torch/torch.h>
27#include <torch/script.h>
69#define OFFSET(x) offsetof(THOptions, x)
70#define FLAGS AV_OPT_FLAG_FILTERING_PARAM
128 if (!model || !*model)
131 th_model = (
THModel *)(*model);
172static void cuda_tensor_deleter(
void *
arg)
199 int stride_bytes =
frame->linesize[0];
203 torch::Tensor byte_tensor = torch::from_blob(
205 {1, height, width, channels},
206 {(long)(height * stride_bytes), (long)stride_bytes,
209 torch::TensorOptions().dtype(torch::kUInt8).device(torch::kCUDA));
213 byte_tensor.to(torch::kFloat32).div(255.0f)
214 .permute({0, 3, 1, 2})
221static void fill_model_output_th_cuda(
THModel *th_model,
TaskItem *task, torch::Tensor &out_slice)
229 bool needs_flip =
false;
230 switch (hw_frames_ctx->sw_format) {
232 hw_channels = 3; rgb_start = 0; needs_flip =
false;
235 hw_channels = 3; rgb_start = 0; needs_flip =
true;
238 hw_channels = 4; rgb_start = 0; needs_flip =
false;
241 hw_channels = 4; rgb_start = 0; needs_flip =
true;
244 hw_channels = 4; rgb_start = 1; needs_flip =
false;
247 hw_channels = 4; rgb_start = 1; needs_flip =
true;
251 "Unsupported sw_format for CUDA zero-copy output\n");
257 torch::Tensor out_u8 =
258 out_slice.mul(255.0f)
259 .permute({0, 2, 3, 1})
263 out_u8 = out_u8.flip({3});
265 int out_h = (int)out_u8.size(1);
266 int out_w = (int)out_u8.size(2);
270 torch::Tensor out_frame_tensor = torch::from_blob(
272 {1, out_h, out_w, hw_channels},
273 {(long)(out_h * task->out_frame->linesize[0]),
274 (long)task->out_frame->linesize[0],
275 (long)hw_channels, 1L},
277 torch::TensorOptions()
278 .dtype(torch::kUInt8)
279 .device(torch::kCUDA));
282 out_frame_tensor.slice(3, rgb_start, rgb_start + 3)
286 torch::cuda::synchronize();
297 int ret, width_idx, height_idx, channel_idx;
298 int batch_size =
ctx->batch_size;
299 float *batch_data =
NULL;
328 for (
int i = 0;
i < batch_size;
i++) {
357 infer_request->
output =
new torch::Tensor();
358 *infer_request->
input_tensor = torch::from_blob(batch_data,
379 std::vector<torch::jit::IValue>
inputs;
380 torch::NoGradGuard no_grad;
392 if (
ctx->torch_option.optimize)
393 torch::jit::setGraphExecutorOptimize(
true);
395 torch::jit::setGraphExecutorOptimize(
false);
402 const char *device_name =
ctx->device ?
ctx->device :
"cpu";
403 c10::Device device(device_name);
418 torch::Tensor *output = infer_request->
output;
421 auto slices = torch::split(*output, 1, 0);
425 torch::Tensor out_slice = slices[
i];
426 c10::IntArrayRef
sizes = out_slice.sizes();
432 if (
sizes.size() == 4) {
449 fill_model_output_th_cuda(th_model, task, out_slice);
452 if (out_slice.device() != torch::kCPU)
453 out_slice = out_slice.to(torch::kCPU);
455 outputs.data = out_slice.data_ptr();
488 av_log(th_model->
ctx,
AV_LOG_ERROR,
"Unable to push back request_queue when failed to start inference.\n");
505 if (lltask ==
NULL) {
515 ret = fill_model_input_th_cuda(th_model, request);
551 const char *output_name,
int *output_width,
int *output_height)
559 .input_name = input_name,
560 .output_names = &output_name,
609 const char *device_name =
ctx->device ?
ctx->device :
"cpu";
614 model = &th_model->
model;
617 c10::Device device = c10::Device(device_name);
618 if (device.is_xpu()) {
623#if TORCH_VERSION_MAJOR > 2 || (TORCH_VERSION_MAJOR == 2 && TORCH_VERSION_MINOR >= 6)
624 at::detail::getXPUHooks().init();
626 at::detail::getXPUHooks().initXPU();
628 }
else if (device.is_cuda()) {
630 if (!torch::cuda::is_available()) {
635 }
else if (!device.is_cpu()) {
641 th_model->
jit_model =
new torch::jit::Module;
642 (*th_model->
jit_model) = torch::jit::load(
ctx->model_filename);
644 }
catch (
const c10::Error& e) {
649 if (
ctx->nireq <= 0) {
658 for (
int i = 0;
i <
ctx->nireq;
i++) {
static const AVFilterPad inputs[]
static const AVFilterPad outputs[]
static AVFormatContext * ctx
#define i(width, name, range_min, range_max)
int ff_check_exec_params(void *ctx, DNNBackendType backend, DNNFunctionType func_type, DNNExecBaseParams *exec_params)
void ff_dnn_wait_requests(SafeQueue *request_queue, int nireq)
Wait for all inference requests to complete before teardown.
DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out)
Extract input and output frame from the Task Queue after asynchronous inference.
int ff_dnn_async_module_cleanup(DNNAsyncExecModule *async_module)
Join the Async Execution thread and set module pointers to NULL.
int ff_dnn_fill_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int async, int do_ioproc)
Fill the Task for Backend Execution.
int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module)
Start asynchronous inference routine for the TensorFlow model on a detached thread.
int ff_dnn_fill_gettingoutput_task(TaskItem *task, DNNExecBaseParams *exec_params, void *backend_model, int input_height, int input_width, void *ctx)
Allocate input and output frames and fill the Task with execution parameters.
DNN common functions different backends.
#define DNN_DEFINE_CLASS(fname)
static void infer_completion_callback(void *args)
const DNNModule ff_dnn_backend_torch
static int get_input_th(DNNModel *model, DNNData *input, const char *input_name)
static int fill_model_input_th(THModel *th_model, THRequestItem *request)
static DNNAsyncStatusType dnn_get_result_th(const DNNModel *model, AVFrame **in, AVFrame **out)
static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue)
static int th_start_inference(void *args)
static int execute_model_th(THRequestItem *request, Queue *lltask_queue)
static const AVOption dnn_th_options[]
static void th_free_request(THInferRequest *request)
static THInferRequest * th_create_inference_request(void)
static int dnn_flush_th(const DNNModel *model)
static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_params)
static DNNModel * dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, AVFilterContext *filter_ctx)
static int get_output_th(DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
static void deleter(void *arg)
static void destroy_request_item(THRequestItem **arg)
static void dnn_free_model_th(DNNModel **model)
static void infer_completion_callback(void *args)
static int dnn_get_height_idx_by_layout(DNNLayout layout)
static int dnn_get_width_idx_by_layout(DNNLayout layout)
#define DNN_GENERIC_ERROR
static int dnn_get_channel_idx_by_layout(DNNLayout layout)
int ff_proc_from_frame_to_dnn(AVFrame *frame, DNNData *input, void *log_ctx)
int ff_proc_from_dnn_to_frame(AVFrame *frame, DNNData *output, void *log_ctx)
DNN input&output process between AVFrame and DNNData.
static const uint8_t frame_size[4]
@ AV_OPT_TYPE_INT
Underlying C type is int.
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
#define AV_LOG_INFO
Standard information.
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
An API-specific header for AV_HWDEVICE_TYPE_CUDA.
FFmpeg internal API for CUDA.
static const int sizes[][2]
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
Memory handling functions.
@ AV_PIX_FMT_RGB24
packed RGB 8:8:8, 24bpp, RGBRGB...
@ AV_PIX_FMT_BGR0
packed BGR 8:8:8, 32bpp, BGRXBGRX... X=unused/undefined
@ AV_PIX_FMT_CUDA
HW acceleration through CUDA.
@ AV_PIX_FMT_0BGR
packed BGR 8:8:8, 32bpp, XBGRXBGR... X=unused/undefined
@ AV_PIX_FMT_RGB0
packed RGB 8:8:8, 32bpp, RGBXRGBX... X=unused/undefined
@ AV_PIX_FMT_BGR24
packed RGB 8:8:8, 24bpp, BGRBGR...
@ AV_PIX_FMT_0RGB
packed RGB 8:8:8, 32bpp, XRGBXRGB... X=unused/undefined
void ff_queue_destroy(Queue *q)
Destroy the Queue instance.
void * ff_queue_pop_front(Queue *q)
Remove and free first element from the Queue.
int ff_queue_push_back(Queue *q, void *v)
Add data to the tail of the queue.
void * ff_queue_peek_front(Queue *q)
Return a pointer to the data at the head of the queue.
size_t ff_queue_size(Queue *q)
Return the length of the Queue.
Queue * ff_queue_create(void)
Create a Queue instance.
int ff_safe_queue_push_back(SafeQueue *sq, void *v)
Add data to the tail of queue in the SafeQueue after locking mutex.
void * ff_safe_queue_pop_front(SafeQueue *sq)
Remove and free first element from the queue in SafeQueue.
size_t ff_safe_queue_size(SafeQueue *sq)
Return the length of the SafeQueue.
SafeQueue * ff_safe_queue_create(void)
Create and initialize a SafeQueue instance.
void ff_safe_queue_destroy(SafeQueue *sq)
Destroy the SafeQueue instance.
uint8_t * data
The data buffer.
This structure describes decoded (raw) audio or video data.
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame.
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames,...
This struct describes a set or pool of "hardware" frames (i.e.
Common Async Execution Mechanism for the DNN Backends.
void * args
Argument for the execution functions.
int(* start_inference)(void *request)
Synchronous inference function for the backend with corresponding request item as the argument.
void(* callback)(void *args)
Completion Callback for the backend.
int(* get_input)(struct DNNModel *model, DNNData *input, const char *input_name)
int(* get_output)(struct DNNModel *model, const char *input_name, int input_width, int input_height, const char *output_name, int *output_width, int *output_height)
FramePrePostProc frame_pre_proc
FramePrePostProc frame_post_proc
AVFilterContext * filter_ctx
DNNFunctionType func_type
Linear double-ended data structure.
Double-ended queue with mutex locks ensuring data consistency while multithreading.
torch::Tensor * input_tensor
torch::jit::Module * jit_model
SafeQueue * request_queue
LastLevelTaskItem ** lltasks
THInferRequest * infer_request
DNNAsyncExecModule exec_module
#define av_malloc_array(a, b)
static FilteringContext * filter_ctx