FFmpeg
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
vda_h264.c
Go to the documentation of this file.
1 /*
2  * VDA H264 HW acceleration.
3  *
4  * copyright (c) 2011 Sebastien Zwickert
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22 
23 #include <CoreFoundation/CFDictionary.h>
24 #include <CoreFoundation/CFNumber.h>
25 #include <CoreFoundation/CFData.h>
26 
27 #include "vda.h"
28 #include "libavutil/avutil.h"
29 #include "h264dec.h"
30 
31 struct vda_buffer {
32  CVPixelBufferRef cv_buffer;
33 };
34 #include "internal.h"
35 #include "vda_vt_internal.h"
36 
37 /* Decoder callback that adds the vda frame to the queue in display order. */
38 static void vda_decoder_callback(void *vda_hw_ctx,
39  CFDictionaryRef user_info,
40  OSStatus status,
41  uint32_t infoFlags,
42  CVImageBufferRef image_buffer)
43 {
44  struct vda_context *vda_ctx = vda_hw_ctx;
45 
46  if (infoFlags & kVDADecodeInfo_FrameDropped)
47  vda_ctx->cv_buffer = NULL;
48 
49  if (!image_buffer)
50  return;
51 
52  if (vda_ctx->cv_pix_fmt_type != CVPixelBufferGetPixelFormatType(image_buffer))
53  return;
54 
55  vda_ctx->cv_buffer = CVPixelBufferRetain(image_buffer);
56 }
57 
58 static int vda_sync_decode(VTContext *ctx, struct vda_context *vda_ctx)
59 {
60  OSStatus status;
61  CFDataRef coded_frame;
62  uint32_t flush_flags = 1 << 0; ///< kVDADecoderFlush_emitFrames
63 
64  coded_frame = CFDataCreate(kCFAllocatorDefault,
65  ctx->bitstream,
66  ctx->bitstream_size);
67 
68  status = VDADecoderDecode(vda_ctx->decoder, 0, coded_frame, NULL);
69 
70  if (kVDADecoderNoErr == status)
71  status = VDADecoderFlush(vda_ctx->decoder, flush_flags);
72 
73  CFRelease(coded_frame);
74 
75  return status;
76 }
77 
78 
80  av_unused const uint8_t *buffer,
81  av_unused uint32_t size)
82 {
83  VTContext *vda = avctx->internal->hwaccel_priv_data;
84  struct vda_context *vda_ctx = avctx->hwaccel_context;
85 
86  if (!vda_ctx->decoder)
87  return -1;
88 
89  vda->bitstream_size = 0;
90 
91  return 0;
92 }
93 
95  const uint8_t *buffer,
96  uint32_t size)
97 {
98  VTContext *vda = avctx->internal->hwaccel_priv_data;
99  struct vda_context *vda_ctx = avctx->hwaccel_context;
100  void *tmp;
101 
102  if (!vda_ctx->decoder)
103  return -1;
104 
105  tmp = av_fast_realloc(vda->bitstream,
106  &vda->allocated_size,
107  vda->bitstream_size + size + 4);
108  if (!tmp)
109  return AVERROR(ENOMEM);
110 
111  vda->bitstream = tmp;
112 
113  AV_WB32(vda->bitstream + vda->bitstream_size, size);
114  memcpy(vda->bitstream + vda->bitstream_size + 4, buffer, size);
115 
116  vda->bitstream_size += size + 4;
117 
118  return 0;
119 }
120 
121 static void vda_h264_release_buffer(void *opaque, uint8_t *data)
122 {
123  struct vda_buffer *context = opaque;
124  CVPixelBufferRelease(context->cv_buffer);
125  av_free(context);
126 }
127 
129 {
130  H264Context *h = avctx->priv_data;
131  VTContext *vda = avctx->internal->hwaccel_priv_data;
132  struct vda_context *vda_ctx = avctx->hwaccel_context;
133  AVFrame *frame = h->cur_pic_ptr->f;
134  struct vda_buffer *context;
136  int status;
137 
138  if (!vda_ctx->decoder || !vda->bitstream)
139  return -1;
140 
141  status = vda_sync_decode(vda, vda_ctx);
142  frame->data[3] = (void*)vda_ctx->cv_buffer;
143 
144  if (status)
145  av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
146 
147  if (!vda_ctx->use_ref_buffer || status)
148  return status;
149 
150  context = av_mallocz(sizeof(*context));
151  buffer = av_buffer_create(NULL, 0, vda_h264_release_buffer, context, 0);
152  if (!context || !buffer) {
153  CVPixelBufferRelease(vda_ctx->cv_buffer);
154  av_free(context);
155  return -1;
156  }
157 
158  context->cv_buffer = vda_ctx->cv_buffer;
159  frame->buf[3] = buffer;
160 
161  return status;
162 }
163 
164 int ff_vda_create_decoder(struct vda_context *vda_ctx,
165  uint8_t *extradata,
166  int extradata_size)
167 {
168  OSStatus status;
169  CFNumberRef height;
170  CFNumberRef width;
171  CFNumberRef format;
172  CFDataRef avc_data;
173  CFMutableDictionaryRef config_info;
174  CFMutableDictionaryRef buffer_attributes;
175  CFMutableDictionaryRef io_surface_properties;
176  CFNumberRef cv_pix_fmt;
177 
178  vda_ctx->priv_bitstream = NULL;
179  vda_ctx->priv_allocated_size = 0;
180 
181  /* Each VCL NAL in the bitstream sent to the decoder
182  * is preceded by a 4 bytes length header.
183  * Change the avcC atom header if needed, to signal headers of 4 bytes. */
184  if (extradata_size >= 4 && (extradata[4] & 0x03) != 0x03) {
185  uint8_t *rw_extradata;
186 
187  if (!(rw_extradata = av_malloc(extradata_size)))
188  return AVERROR(ENOMEM);
189 
190  memcpy(rw_extradata, extradata, extradata_size);
191 
192  rw_extradata[4] |= 0x03;
193 
194  avc_data = CFDataCreate(kCFAllocatorDefault, rw_extradata, extradata_size);
195 
196  av_freep(&rw_extradata);
197  } else {
198  avc_data = CFDataCreate(kCFAllocatorDefault, extradata, extradata_size);
199  }
200 
201  config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
202  4,
203  &kCFTypeDictionaryKeyCallBacks,
204  &kCFTypeDictionaryValueCallBacks);
205 
206  height = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->height);
207  width = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->width);
208  format = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &vda_ctx->format);
209 
210  CFDictionarySetValue(config_info, kVDADecoderConfiguration_Height, height);
211  CFDictionarySetValue(config_info, kVDADecoderConfiguration_Width, width);
212  CFDictionarySetValue(config_info, kVDADecoderConfiguration_SourceFormat, format);
213  CFDictionarySetValue(config_info, kVDADecoderConfiguration_avcCData, avc_data);
214 
215  buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
216  2,
217  &kCFTypeDictionaryKeyCallBacks,
218  &kCFTypeDictionaryValueCallBacks);
219  io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
220  0,
221  &kCFTypeDictionaryKeyCallBacks,
222  &kCFTypeDictionaryValueCallBacks);
223  cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault,
224  kCFNumberSInt32Type,
225  &vda_ctx->cv_pix_fmt_type);
226  CFDictionarySetValue(buffer_attributes,
227  kCVPixelBufferPixelFormatTypeKey,
228  cv_pix_fmt);
229  CFDictionarySetValue(buffer_attributes,
230  kCVPixelBufferIOSurfacePropertiesKey,
231  io_surface_properties);
232 
233  status = VDADecoderCreate(config_info,
234  buffer_attributes,
235  (VDADecoderOutputCallback *)vda_decoder_callback,
236  vda_ctx,
237  &vda_ctx->decoder);
238 
239  CFRelease(height);
240  CFRelease(width);
241  CFRelease(format);
242  CFRelease(avc_data);
243  CFRelease(config_info);
244  CFRelease(io_surface_properties);
245  CFRelease(cv_pix_fmt);
246  CFRelease(buffer_attributes);
247 
248  return status;
249 }
250 
252 {
253  OSStatus status = kVDADecoderNoErr;
254 
255  if (vda_ctx->decoder)
256  status = VDADecoderDestroy(vda_ctx->decoder);
257 
258  return status;
259 }
260 
262  .name = "h264_vda",
263  .type = AVMEDIA_TYPE_VIDEO,
264  .id = AV_CODEC_ID_H264,
265  .pix_fmt = AV_PIX_FMT_VDA_VLD,
266  .start_frame = vda_old_h264_start_frame,
267  .decode_slice = vda_old_h264_decode_slice,
268  .end_frame = vda_old_h264_end_frame,
269  .uninit = ff_videotoolbox_uninit,
270  .priv_data_size = sizeof(VTContext),
271 };
272 
273 void ff_vda_output_callback(void *opaque,
274  CFDictionaryRef user_info,
275  OSStatus status,
276  uint32_t infoFlags,
277  CVImageBufferRef image_buffer)
278 {
279  AVCodecContext *ctx = opaque;
280  VTContext *vda = ctx->internal->hwaccel_priv_data;
281 
282 
283  if (vda->frame) {
284  CVPixelBufferRelease(vda->frame);
285  vda->frame = NULL;
286  }
287 
288  if (!image_buffer)
289  return;
290 
291  vda->frame = CVPixelBufferRetain(image_buffer);
292 }
293 
295 {
296  H264Context *h = avctx->priv_data;
297  VTContext *vda = avctx->internal->hwaccel_priv_data;
298  AVVDAContext *vda_ctx = avctx->hwaccel_context;
299  AVFrame *frame = h->cur_pic_ptr->f;
300  uint32_t flush_flags = 1 << 0; ///< kVDADecoderFlush_emitFrames
301  CFDataRef coded_frame;
302  OSStatus status;
303 
304  if (!vda->bitstream_size)
305  return AVERROR_INVALIDDATA;
306 
307 
308  coded_frame = CFDataCreate(kCFAllocatorDefault,
309  vda->bitstream,
310  vda->bitstream_size);
311 
312  status = VDADecoderDecode(vda_ctx->decoder, 0, coded_frame, NULL);
313 
314  if (status == kVDADecoderNoErr)
315  status = VDADecoderFlush(vda_ctx->decoder, flush_flags);
316 
317  CFRelease(coded_frame);
318 
319  if (!vda->frame)
320  return AVERROR_UNKNOWN;
321 
322  if (status != kVDADecoderNoErr) {
323  av_log(avctx, AV_LOG_ERROR, "Failed to decode frame (%d)\n", status);
324  return AVERROR_UNKNOWN;
325  }
326 
327  return ff_videotoolbox_buffer_create(vda, frame);
328 }
329 
331 {
332  AVVDAContext *vda_ctx = avctx->hwaccel_context;
333  OSStatus status = kVDADecoderNoErr;
334  CFNumberRef height;
335  CFNumberRef width;
336  CFNumberRef format;
337  CFDataRef avc_data;
338  CFMutableDictionaryRef config_info;
339  CFMutableDictionaryRef buffer_attributes;
340  CFMutableDictionaryRef io_surface_properties;
341  CFNumberRef cv_pix_fmt;
342  int32_t fmt = 'avc1', pix_fmt = vda_ctx->cv_pix_fmt_type;
343 
344  // kCVPixelFormatType_420YpCbCr8Planar;
345 
346  avc_data = ff_videotoolbox_avcc_extradata_create(avctx);
347 
348  config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
349  4,
350  &kCFTypeDictionaryKeyCallBacks,
351  &kCFTypeDictionaryValueCallBacks);
352 
353  height = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &avctx->height);
354  width = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &avctx->width);
355  format = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &fmt);
356  CFDictionarySetValue(config_info, kVDADecoderConfiguration_Height, height);
357  CFDictionarySetValue(config_info, kVDADecoderConfiguration_Width, width);
358  CFDictionarySetValue(config_info, kVDADecoderConfiguration_avcCData, avc_data);
359  CFDictionarySetValue(config_info, kVDADecoderConfiguration_SourceFormat, format);
360 
361  buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
362  2,
363  &kCFTypeDictionaryKeyCallBacks,
364  &kCFTypeDictionaryValueCallBacks);
365  io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
366  0,
367  &kCFTypeDictionaryKeyCallBacks,
368  &kCFTypeDictionaryValueCallBacks);
369  cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault,
370  kCFNumberSInt32Type,
371  &pix_fmt);
372 
373  CFDictionarySetValue(buffer_attributes,
374  kCVPixelBufferPixelFormatTypeKey,
375  cv_pix_fmt);
376  CFDictionarySetValue(buffer_attributes,
377  kCVPixelBufferIOSurfacePropertiesKey,
378  io_surface_properties);
379 
380  status = VDADecoderCreate(config_info,
381  buffer_attributes,
382  (VDADecoderOutputCallback *)ff_vda_output_callback,
383  avctx,
384  &vda_ctx->decoder);
385 
386  CFRelease(format);
387  CFRelease(height);
388  CFRelease(width);
389  CFRelease(avc_data);
390  CFRelease(config_info);
391  CFRelease(cv_pix_fmt);
392  CFRelease(io_surface_properties);
393  CFRelease(buffer_attributes);
394 
395  if (status != kVDADecoderNoErr) {
396  av_log(avctx, AV_LOG_ERROR, "Cannot initialize VDA %d\n", status);
397  }
398 
399  switch (status) {
400  case kVDADecoderHardwareNotSupportedErr:
401  case kVDADecoderFormatNotSupportedErr:
402  return AVERROR(ENOSYS);
403  case kVDADecoderConfigurationError:
404  return AVERROR(EINVAL);
405  case kVDADecoderDecoderFailedErr:
406  return AVERROR_INVALIDDATA;
407  case kVDADecoderNoErr:
408  return 0;
409  default:
410  return AVERROR_UNKNOWN;
411  }
412 }
413 
415  .name = "h264_vda",
416  .type = AVMEDIA_TYPE_VIDEO,
417  .id = AV_CODEC_ID_H264,
418  .pix_fmt = AV_PIX_FMT_VDA,
419  .alloc_frame = ff_videotoolbox_alloc_frame,
420  .start_frame = ff_videotoolbox_h264_start_frame,
421  .decode_slice = ff_videotoolbox_h264_decode_slice,
422  .end_frame = vda_h264_end_frame,
423  .uninit = ff_videotoolbox_uninit,
424  .priv_data_size = sizeof(VTContext),
425 };
#define NULL
Definition: coverity.c:32
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
static enum AVPixelFormat pix_fmt
This structure describes decoded (raw) audio or video data.
Definition: frame.h:201
ptrdiff_t const GLvoid * data
Definition: opengl_enc.c:101
const char * fmt
Definition: avisynth_c.h:769
uint8_t * bitstream
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:393
AVHWAccel ff_h264_vda_old_hwaccel
Definition: vda_h264.c:261
void ff_vda_output_callback(void *opaque, CFDictionaryRef user_info, OSStatus status, uint32_t infoFlags, CVImageBufferRef image_buffer)
Definition: vda_h264.c:273
Convenience header that includes libavutil's core.
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:222
H264Context.
Definition: h264dec.h:337
AVFrame * f
Definition: h264dec.h:129
static int vda_old_h264_end_frame(AVCodecContext *avctx)
Definition: vda_h264.c:128
int format
The frame format.
Definition: vda.h:112
OSType cv_pix_fmt_type
The pixel format for output image buffers.
Definition: vda.h:120
This struct holds all the information that needs to be passed between the caller and libavcodec for i...
Definition: vda.h:163
int use_ref_buffer
Use av_buffer to manage buffer.
Definition: vda.h:146
CFDataRef ff_videotoolbox_avcc_extradata_create(AVCodecContext *avctx)
Definition: videotoolbox.c:90
uint8_t
#define av_malloc(s)
void * hwaccel_context
Hardware accelerator context.
Definition: avcodec.h:3094
OSType cv_pix_fmt_type
CVPixelBuffer Format Type that VDA will use for decoded frames; set by the caller.
Definition: vda.h:179
int ff_videotoolbox_uninit(AVCodecContext *avctx)
Definition: videotoolbox.c:343
static AVFrame * frame
#define height
int ff_videotoolbox_alloc_frame(AVCodecContext *avctx, AVFrame *frame)
Definition: videotoolbox.c:75
int width
The frame width.
Definition: vda.h:96
static int vda_sync_decode(VTContext *ctx, struct vda_context *vda_ctx)
Definition: vda_h264.c:58
ptrdiff_t size
Definition: opengl_enc.c:101
#define av_log(a,...)
CVPixelBufferRef cv_buffer
The Core Video pixel buffer that contains the current image data.
Definition: vda.h:80
int ff_vda_default_init(AVCodecContext *avctx)
Definition: vda_h264.c:330
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
VDADecoder decoder
VDA decoder object.
Definition: vda.h:72
#define AVERROR(e)
Definition: error.h:43
int allocated_size
uint16_t width
Definition: gdv.c:47
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:28
HW acceleration through VDA, data[3] contains a CVPixelBufferRef.
Definition: pixfmt.h:224
This structure is used to provide the necessary configurations and data to the VDA FFmpeg HWAccel imp...
Definition: vda.h:65
int ff_videotoolbox_buffer_create(VTContext *vtctx, AVFrame *frame)
Definition: videotoolbox.c:281
const char * name
Name of the hardware accelerated codec.
Definition: avcodec.h:3873
int width
picture width / height.
Definition: avcodec.h:1948
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
H.264 / AVC / MPEG-4 part10 codec.
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given buffer if it is not large enough, otherwise do nothing.
Definition: mem.c:450
int bitstream_size
static void vda_h264_release_buffer(void *opaque, uint8_t *data)
Definition: vda_h264.c:121
VDADecoder decoder
VDA decoder object.
Definition: vda.h:167
int ff_videotoolbox_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size)
Definition: videotoolbox.c:316
Public libavcodec VDA header.
int ff_vda_destroy_decoder(struct vda_context *vda_ctx)
Destroy the video decoder.
Definition: vda_h264.c:251
main external API structure.
Definition: avcodec.h:1761
static int vda_old_h264_decode_slice(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size)
Definition: vda_h264.c:94
static void vda_decoder_callback(void *vda_hw_ctx, CFDictionaryRef user_info, OSStatus status, uint32_t infoFlags, CVImageBufferRef image_buffer)
Definition: vda_h264.c:38
static const char * format
Definition: movenc.c:47
#define AV_WB32(p, v)
Definition: intreadwrite.h:424
H264Picture * cur_pic_ptr
Definition: h264dec.h:346
AVHWAccel ff_h264_vda_hwaccel
Definition: vda_h264.c:414
uint8_t * priv_bitstream
unused
Definition: vda.h:125
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:215
A reference to a data buffer.
Definition: buffer.h:81
hardware decoding through VDA
Definition: pixfmt.h:179
CVPixelBufferRef cv_buffer
Definition: vda_h264.c:32
common internal api header.
static int vda_h264_end_frame(AVCodecContext *avctx)
Definition: vda_h264.c:294
if(ret< 0)
Definition: vf_mcdeint.c:279
void * hwaccel_priv_data
hwaccel-specific private data
Definition: internal.h:192
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:71
void * priv_data
Definition: avcodec.h:1803
#define av_free(p)
struct AVCodecInternal * internal
Private context used for internal data.
Definition: avcodec.h:1811
int ff_vda_create_decoder(struct vda_context *vda_ctx, uint8_t *extradata, int extradata_size)
Create the video decoder.
Definition: vda_h264.c:164
static int vda_old_h264_start_frame(AVCodecContext *avctx, av_unused const uint8_t *buffer, av_unused uint32_t size)
Definition: vda_h264.c:79
int ff_videotoolbox_h264_start_frame(AVCodecContext *avctx, const uint8_t *buffer, uint32_t size)
Definition: videotoolbox.c:300
int priv_allocated_size
unused
Definition: vda.h:135
#define av_freep(p)
int height
The frame height.
Definition: vda.h:104
GLuint buffer
Definition: opengl_enc.c:102
CVImageBufferRef frame
#define av_unused
Definition: attributes.h:125
static uint8_t tmp[11]
Definition: aes_ctr.c:26