24 #include "config_components.h"
38 #include <Availability.h>
39 #include <AvailabilityMacros.h>
40 #include <TargetConditionals.h>
42 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
43 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
45 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder
46 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder")
49 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
53 #if !HAVE_KCMVIDEOCODECTYPE_VP9
57 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12
68 CVPixelBufferRelease(
ref->pixbuf);
104 frame->crop_right = 0;
105 frame->crop_left = 0;
107 frame->crop_bottom = 0;
112 frame->data[3] = (uint8_t*)
ref->pixbuf;
114 if (
ref->hw_frames_ctx) {
117 if (!
frame->hw_frames_ctx)
154 #define AV_W8(p, v) *(p) = (v)
162 for (
i = 0;
i < src_size;
i++) {
163 if (
i + 2 < src_size &&
165 src[
i + 1] == 0x00 &&
166 src[
i + 2] <= 0x03) {
194 int vt_extradata_size;
195 uint8_t *vt_extradata;
197 vt_extradata_size = 6 + 2 + sps_size + 3 + pps_size;
198 vt_extradata =
av_malloc(vt_extradata_size);
206 AV_W8(p + 1,
h->ps.sps->data[1]);
207 AV_W8(p + 2,
h->ps.sps->data[2]);
208 AV_W8(p + 3,
h->ps.sps->data[3]);
213 p +=
escape_ps(p,
h->ps.sps->data,
h->ps.sps->data_size);
217 p +=
escape_ps(p,
h->ps.pps->data,
h->ps.pps->data_size);
219 av_assert0(p - vt_extradata == vt_extradata_size);
224 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
226 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
234 int i, num_vps = 0, num_sps = 0, num_pps = 0;
240 uint8_t parallelismType;
243 int vt_extradata_size = 23 + 3 + 3 + 3;
244 uint8_t *vt_extradata;
246 #define COUNT_SIZE_PS(T, t) \
247 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
248 if (h->ps.t##ps_list[i]) { \
249 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
250 vt_extradata_size += 2 + lps->data_size; \
259 vt_extradata =
av_malloc(vt_extradata_size);
303 else if (
pps->entropy_coding_sync_enabled_flag &&
pps->tiles_enabled_flag)
305 else if (
pps->entropy_coding_sync_enabled_flag)
307 else if (
pps->tiles_enabled_flag)
311 AV_W8(p + 15, 0xfc | parallelismType);
317 AV_W8(p + 16,
sps->chroma_format_idc | 0xfc);
323 AV_W8(p + 17, (
sps->bit_depth - 8) | 0xfc);
329 AV_W8(p + 18, (
sps->bit_depth_chroma - 8) | 0xfc);
340 AV_W8(p + 21, 0 << 6 |
341 sps->max_sub_layers << 3 |
342 sps->temporal_id_nesting_flag << 2 |
350 #define APPEND_PS(T, t) \
357 HEVC_NAL_##T##PS & 0x3f); \
359 AV_WB16(p + 1, num_##t##ps); \
361 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \
362 if (h->ps.t##ps_list[i]) { \
363 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \
365 AV_WB16(p, lps->data_size); \
367 memcpy(p + 2, lps->data, lps->data_size); \
368 p += 2 + lps->data_size; \
376 av_assert0(p - vt_extradata == vt_extradata_size);
378 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
390 if (
h->is_avc == 1) {
407 memcpy(vtctx->
sps,
h->ps.sps->data + 1, 3);
455 #if CONFIG_VIDEOTOOLBOX
482 VTDecompressionSessionInvalidate(videotoolbox->
session);
483 CFRelease(videotoolbox->
session);
496 CVPixelBufferRelease(vtctx->
frame);
499 videotoolbox_stop(avctx);
510 CVPixelBufferRef pixbuf = (CVPixelBufferRef)vtctx->
frame;
511 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
513 int width = CVPixelBufferGetWidth(pixbuf);
514 int height = CVPixelBufferGetHeight(pixbuf);
528 CVPixelBufferRelease(
ref->pixbuf);
564 if (!
ref->hw_frames_ctx)
570 static void videotoolbox_write_mp4_descr_length(
PutByteContext *pb,
int length)
575 for (
i = 3;
i >= 0;
i--) {
576 b = (length >> (
i * 7)) & 0x7F;
580 bytestream2_put_byteu(pb,
b);
584 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
587 uint8_t *rw_extradata;
598 bytestream2_put_byteu(&pb, 0);
602 bytestream2_put_byteu(&pb, 0x03);
603 videotoolbox_write_mp4_descr_length(&pb, full_size);
605 bytestream2_put_byteu(&pb, 0);
608 bytestream2_put_byteu(&pb, 0x04);
609 videotoolbox_write_mp4_descr_length(&pb, config_size);
610 bytestream2_put_byteu(&pb, 32);
611 bytestream2_put_byteu(&pb, 0x11);
617 bytestream2_put_byteu(&pb, 0x05);
623 bytestream2_put_byteu(&pb, 0x06);
624 bytestream2_put_byteu(&pb, 0x01);
625 bytestream2_put_byteu(&pb, 0x02);
629 data = CFDataCreate(kCFAllocatorDefault, rw_extradata,
s);
635 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
640 CMBlockBufferRef block_buf;
641 CMSampleBufferRef sample_buf;
646 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
657 status = CMSampleBufferCreate(kCFAllocatorDefault,
672 CFRelease(block_buf);
677 static void videotoolbox_decoder_callback(
void *opaque,
678 void *sourceFrameRefCon,
680 VTDecodeInfoFlags
flags,
681 CVImageBufferRef image_buffer,
689 CVPixelBufferRelease(vtctx->
frame);
698 vtctx->
frame = CVPixelBufferRetain(image_buffer);
701 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
704 CMSampleBufferRef sample_buf;
708 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
715 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
721 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
723 CFRelease(sample_buf);
728 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
729 CFDictionaryRef decoder_spec,
733 CMFormatDescriptionRef cm_fmt_desc;
736 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
749 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
753 CFMutableDictionaryRef buffer_attributes;
754 CFMutableDictionaryRef io_surface_properties;
755 CFNumberRef cv_pix_fmt;
759 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
760 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
761 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
pix_fmt);
763 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
765 &kCFTypeDictionaryKeyCallBacks,
766 &kCFTypeDictionaryValueCallBacks);
767 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
769 &kCFTypeDictionaryKeyCallBacks,
770 &kCFTypeDictionaryValueCallBacks);
773 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
774 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
775 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey,
w);
776 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey,
h);
778 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
780 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
783 CFRelease(io_surface_properties);
784 CFRelease(cv_pix_fmt);
788 return buffer_attributes;
791 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
794 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
796 &kCFTypeDictionaryKeyCallBacks,
797 &kCFTypeDictionaryValueCallBacks);
799 CFDictionarySetValue(config_info,
805 CFMutableDictionaryRef avc_info;
808 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
810 &kCFTypeDictionaryKeyCallBacks,
811 &kCFTypeDictionaryValueCallBacks);
814 case kCMVideoCodecType_MPEG4Video :
816 data = videotoolbox_esds_extradata_create(avctx);
818 CFDictionarySetValue(avc_info, CFSTR(
"esds"),
data);
820 case kCMVideoCodecType_H264 :
823 CFDictionarySetValue(avc_info, CFSTR(
"avcC"),
data);
828 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"),
data);
830 #if CONFIG_VP9_VIDEOTOOLBOX_HWACCEL
834 CFDictionarySetValue(avc_info, CFSTR(
"vpcC"),
data);
841 CFDictionarySetValue(config_info,
842 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
856 VTDecompressionOutputCallbackRecord decoder_cb;
857 CFDictionaryRef decoder_spec;
858 CFDictionaryRef buf_attr;
889 case MKTAG(
'a',
'p',
'c',
'o'):
890 case
MKTAG(
'a',
'p',
'c',
's'):
891 case
MKTAG(
'a',
'p',
'c',
'n'):
892 case
MKTAG(
'a',
'p',
'c',
'h'):
893 case
MKTAG(
'a',
'p',
'4',
'h'):
894 case
MKTAG(
'a',
'p',
'4',
'x'):
895 videotoolbox->cm_codec_type =
av_bswap32(avctx->codec_tag);
906 #if defined(MAC_OS_X_VERSION_10_9) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_9) && AV_HAS_BUILTIN(__builtin_available)
908 if (__builtin_available(macOS 10.9, *)) {
909 VTRegisterProfessionalVideoWorkflowVideoDecoders();
914 #if defined(MAC_OS_VERSION_11_0) && !TARGET_OS_IPHONE && (MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_VERSION_11_0) && AV_HAS_BUILTIN(__builtin_available)
915 if (__builtin_available(macOS 11.0, *)) {
916 VTRegisterSupplementalVideoDecoderIfAvailable(videotoolbox->
cm_codec_type);
920 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
933 CFRelease(decoder_spec);
939 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
943 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
944 decoder_cb.decompressionOutputRefCon = avctx;
954 CFRelease(decoder_spec);
959 case kVTVideoDecoderNotAvailableNowErr:
962 case kVTVideoDecoderUnsupportedDataFormatErr:
965 case kVTCouldNotFindVideoDecoderErr:
968 case kVTVideoDecoderMalfunctionErr:
971 case kVTVideoDecoderBadDataErr:
982 static const char *videotoolbox_error_string(OSStatus
status)
985 case kVTVideoDecoderBadDataErr:
987 case kVTVideoDecoderMalfunctionErr:
988 return "decoder malfunction";
989 case kVTInvalidSessionErr:
990 return "invalid session";
1004 videotoolbox_stop(avctx);
1005 if (videotoolbox_start(avctx) != 0) {
1013 status = videotoolbox_session_decode_frame(avctx);
1015 if (
status == kVTVideoDecoderMalfunctionErr ||
status == kVTInvalidSessionErr)
1021 if (!vtctx->
frame) {
1026 return videotoolbox_buffer_create(avctx,
frame);
1054 static int videotoolbox_hevc_decode_params(
AVCodecContext *avctx,
1068 h->output_frame->crop_right = 0;
1069 h->output_frame->crop_left = 0;
1070 h->output_frame->crop_top = 0;
1071 h->output_frame->crop_bottom = 0;
1102 static int videotoolbox_prores_start_frame(
AVCodecContext *avctx,
1109 static int videotoolbox_prores_decode_slice(
AVCodecContext *avctx,
1136 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR16BIPLANARVIDEORANGE
1141 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR10BIPLANARVIDEORANGE
1143 #if HAVE_KCVPIXELFORMATTYPE_444YPCBCR8BIPLANARVIDEORANGE
1150 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR10BIPLANARVIDEORANGE
1152 #if HAVE_KCVPIXELFORMATTYPE_422YPCBCR8BIPLANARVIDEORANGE
1159 #if HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
1180 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1201 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1225 "Failed to map underlying FFmpeg pixel format %s (%s range) to "
1226 "a VideoToolbox format!\n",
1227 attempted_format ? attempted_format->
name :
"<unknown>",
1233 err = videotoolbox_start(avctx);
1252 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1258 .
name =
"h263_videotoolbox",
1263 .start_frame = videotoolbox_mpeg_start_frame,
1264 .decode_slice = videotoolbox_mpeg_decode_slice,
1265 .end_frame = videotoolbox_mpeg_end_frame,
1273 .
name =
"hevc_videotoolbox",
1278 .start_frame = videotoolbox_hevc_start_frame,
1279 .decode_slice = videotoolbox_hevc_decode_slice,
1280 .decode_params = videotoolbox_hevc_decode_params,
1281 .end_frame = videotoolbox_hevc_end_frame,
1289 .
name =
"h264_videotoolbox",
1297 .end_frame = videotoolbox_h264_end_frame,
1305 .
name =
"mpeg1_videotoolbox",
1310 .start_frame = videotoolbox_mpeg_start_frame,
1311 .decode_slice = videotoolbox_mpeg_decode_slice,
1312 .end_frame = videotoolbox_mpeg_end_frame,
1320 .
name =
"mpeg2_videotoolbox",
1325 .start_frame = videotoolbox_mpeg_start_frame,
1326 .decode_slice = videotoolbox_mpeg_decode_slice,
1327 .end_frame = videotoolbox_mpeg_end_frame,
1335 .
name =
"mpeg4_videotoolbox",
1340 .start_frame = videotoolbox_mpeg_start_frame,
1341 .decode_slice = videotoolbox_mpeg_decode_slice,
1342 .end_frame = videotoolbox_mpeg_end_frame,
1350 .
name =
"prores_videotoolbox",
1355 .start_frame = videotoolbox_prores_start_frame,
1356 .decode_slice = videotoolbox_prores_decode_slice,
1357 .end_frame = videotoolbox_prores_end_frame,
1370 ret->output_callback = videotoolbox_decoder_callback;
1373 if (cv_pix_fmt_type == 0) {
1374 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1376 ret->cv_pix_fmt_type = cv_pix_fmt_type;
1384 return av_videotoolbox_alloc_context_with_pix_fmt(
AV_PIX_FMT_NONE,
false);
1399 return videotoolbox_start(avctx);
1405 videotoolbox_stop(avctx);