35 #include <TargetConditionals.h> 37 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder 38 # define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder") 40 #ifndef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder 41 # define kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder CFSTR("EnableHardwareAcceleratedVideoDecoder") 44 #if !HAVE_KCMVIDEOCODECTYPE_HEVC 48 #define VIDEOTOOLBOX_ESDS_EXTRADATA_PADDING 12 59 CVPixelBufferRelease(ref->
pixbuf);
141 #define AV_W8(p, v) *(p) = (v) 170 av_assert0(p - vt_extradata == vt_extradata_size);
177 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
185 int i, num_vps = 0, num_sps = 0, num_pps = 0;
194 int vt_extradata_size = 23 + 3 + 3 + 3;
197 #define COUNT_SIZE_PS(T, t) \ 198 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \ 199 if (h->ps.t##ps_list[i]) { \ 200 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \ 201 vt_extradata_size += 2 + lps->data_size; \ 210 vt_extradata =
av_malloc(vt_extradata_size);
262 AV_W8(p + 15, 0xfc | parallelismType);
291 AV_W8(p + 21, 0 << 6 |
301 #define APPEND_PS(T, t) \ 308 HEVC_NAL_##T##PS & 0x3f); \ 310 AV_WB16(p + 1, num_##t##ps); \ 312 for (i = 0; i < HEVC_MAX_##T##PS_COUNT; i++) { \ 313 if (h->ps.t##ps_list[i]) { \ 314 const HEVC##T##PS *lps = (const HEVC##T##PS *)h->ps.t##ps_list[i]->data; \ 316 AV_WB16(p, lps->data_size); \ 318 memcpy(p + 2, lps->data, lps->data_size); \ 319 p += 2 + lps->data_size; \ 327 av_assert0(p - vt_extradata == vt_extradata_size);
329 data = CFDataCreate(kCFAllocatorDefault, vt_extradata, vt_extradata_size);
361 if (size > 4 && memcmp(vtctx->
sps, buffer + 1, 3) != 0) {
363 memcpy(vtctx->
sps, buffer + 1, 3);
412 CVPixelBufferRelease(vtctx->
frame);
418 #if CONFIG_VIDEOTOOLBOX 436 CVPixelBufferRef
pixbuf = (CVPixelBufferRef)vtctx->
frame;
437 OSType pixel_format = CVPixelBufferGetPixelFormatType(pixbuf);
439 int width = CVPixelBufferGetWidth(pixbuf);
440 int height = CVPixelBufferGetHeight(pixbuf);
445 if (!frame->
buf[0] || frame->
data[3]) {
454 CVPixelBufferRelease(ref->
pixbuf);
465 cached_frames->
width != width ||
466 cached_frames->
height != height) {
475 hw_frames->
width = width;
476 hw_frames->
height = height;
501 for (i = 3; i >= 0; i--) {
502 b = (length >> (i * 7)) & 0x7F;
506 bytestream2_put_byteu(pb, b);
510 static CFDataRef videotoolbox_esds_extradata_create(
AVCodecContext *avctx)
524 bytestream2_put_byteu(&pb, 0);
528 bytestream2_put_byteu(&pb, 0x03);
529 videotoolbox_write_mp4_descr_length(&pb, full_size);
531 bytestream2_put_byteu(&pb, 0);
534 bytestream2_put_byteu(&pb, 0x04);
535 videotoolbox_write_mp4_descr_length(&pb, config_size);
536 bytestream2_put_byteu(&pb, 32);
537 bytestream2_put_byteu(&pb, 0x11);
543 bytestream2_put_byteu(&pb, 0x05);
549 bytestream2_put_byteu(&pb, 0x06);
550 bytestream2_put_byteu(&pb, 0x01);
551 bytestream2_put_byteu(&pb, 0x02);
555 data = CFDataCreate(kCFAllocatorDefault, rw_extradata, s);
561 static CMSampleBufferRef videotoolbox_sample_buffer_create(CMFormatDescriptionRef fmt_desc,
566 CMBlockBufferRef block_buf;
567 CMSampleBufferRef sample_buf;
572 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,
583 status = CMSampleBufferCreate(kCFAllocatorDefault,
598 CFRelease(block_buf);
603 static void videotoolbox_decoder_callback(
void *opaque,
604 void *sourceFrameRefCon,
606 VTDecodeInfoFlags
flags,
607 CVImageBufferRef image_buffer,
615 CVPixelBufferRelease(vtctx->
frame);
624 vtctx->
frame = CVPixelBufferRetain(image_buffer);
627 static OSStatus videotoolbox_session_decode_frame(
AVCodecContext *avctx)
630 CMSampleBufferRef sample_buf;
634 sample_buf = videotoolbox_sample_buffer_create(videotoolbox->
cm_fmt_desc,
641 status = VTDecompressionSessionDecodeFrame(videotoolbox->
session,
647 status = VTDecompressionSessionWaitForAsynchronousFrames(videotoolbox->
session);
649 CFRelease(sample_buf);
654 static CMVideoFormatDescriptionRef videotoolbox_format_desc_create(CMVideoCodecType
codec_type,
655 CFDictionaryRef decoder_spec,
659 CMFormatDescriptionRef cm_fmt_desc;
662 status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
675 static CFDictionaryRef videotoolbox_buffer_attributes_create(
int width,
679 CFMutableDictionaryRef buffer_attributes;
680 CFMutableDictionaryRef io_surface_properties;
681 CFNumberRef cv_pix_fmt;
685 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
width);
686 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &
height);
687 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
689 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
691 &kCFTypeDictionaryKeyCallBacks,
692 &kCFTypeDictionaryValueCallBacks);
693 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
695 &kCFTypeDictionaryKeyCallBacks,
696 &kCFTypeDictionaryValueCallBacks);
699 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
700 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
701 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
702 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
704 CFDictionarySetValue(buffer_attributes, kCVPixelBufferOpenGLESCompatibilityKey, kCFBooleanTrue);
706 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
709 CFRelease(io_surface_properties);
710 CFRelease(cv_pix_fmt);
714 return buffer_attributes;
717 static CFDictionaryRef videotoolbox_decoder_config_create(CMVideoCodecType
codec_type,
720 CFMutableDictionaryRef config_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
722 &kCFTypeDictionaryKeyCallBacks,
723 &kCFTypeDictionaryValueCallBacks);
725 CFDictionarySetValue(config_info,
731 CFMutableDictionaryRef avc_info;
734 avc_info = CFDictionaryCreateMutable(kCFAllocatorDefault,
736 &kCFTypeDictionaryKeyCallBacks,
737 &kCFTypeDictionaryValueCallBacks);
740 case kCMVideoCodecType_MPEG4Video :
742 data = videotoolbox_esds_extradata_create(avctx);
744 CFDictionarySetValue(avc_info, CFSTR(
"esds"), data);
746 case kCMVideoCodecType_H264 :
749 CFDictionarySetValue(avc_info, CFSTR(
"avcC"), data);
754 CFDictionarySetValue(avc_info, CFSTR(
"hvcC"), data);
760 CFDictionarySetValue(config_info,
761 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
775 VTDecompressionOutputCallbackRecord decoder_cb;
776 CFDictionaryRef decoder_spec;
777 CFDictionaryRef buf_attr;
807 decoder_spec = videotoolbox_decoder_config_create(videotoolbox->
cm_codec_type, avctx);
820 CFRelease(decoder_spec);
826 buf_attr = videotoolbox_buffer_attributes_create(avctx->
width,
830 decoder_cb.decompressionOutputCallback = videotoolbox_decoder_callback;
831 decoder_cb.decompressionOutputRefCon = avctx;
833 status = VTDecompressionSessionCreate(
NULL,
841 CFRelease(decoder_spec);
846 case kVTVideoDecoderNotAvailableNowErr:
849 case kVTVideoDecoderUnsupportedDataFormatErr:
852 case kVTCouldNotFindVideoDecoderErr:
855 case kVTVideoDecoderMalfunctionErr:
858 case kVTVideoDecoderBadDataErr:
881 VTDecompressionSessionInvalidate(videotoolbox->
session);
882 CFRelease(videotoolbox->
session);
887 static const char *videotoolbox_error_string(OSStatus
status)
890 case kVTVideoDecoderBadDataErr:
892 case kVTVideoDecoderMalfunctionErr:
893 return "decoder malfunction";
894 case kVTInvalidSessionErr:
895 return "invalid session";
909 videotoolbox_stop(avctx);
910 if (videotoolbox_start(avctx) != 0) {
918 status = videotoolbox_session_decode_frame(avctx);
919 if (status != noErr) {
920 if (status == kVTVideoDecoderMalfunctionErr || status == kVTInvalidSessionErr)
922 av_log(avctx,
AV_LOG_ERROR,
"Failed to decode frame (%s, %d)\n", videotoolbox_error_string(status), (
int)status);
931 return videotoolbox_buffer_create(avctx, frame);
939 int ret = videotoolbox_common_end_frame(avctx, frame);
978 int ret = videotoolbox_common_end_frame(avctx, frame);
1004 return videotoolbox_common_end_frame(avctx, frame);
1016 videotoolbox_stop(avctx);
1049 "Either hw_frames_ctx or hw_device_ctx must be set.\n");
1070 hw_frames->
sw_format = videotoolbox_best_pixel_format(avctx);
1094 "Failed to map underlying FFmpeg pixel format %s (%s range) to " 1095 "a VideoToolbox format!\n",
1096 attempted_format ? attempted_format->
name :
"<unknown>",
1102 err = videotoolbox_start(avctx);
1121 frames_ctx->
sw_format = videotoolbox_best_pixel_format(avctx);
1127 .
name =
"h263_videotoolbox",
1132 .start_frame = videotoolbox_mpeg_start_frame,
1133 .decode_slice = videotoolbox_mpeg_decode_slice,
1134 .end_frame = videotoolbox_mpeg_end_frame,
1135 .frame_params = videotoolbox_frame_params,
1136 .init = videotoolbox_common_init,
1142 .
name =
"hevc_videotoolbox",
1147 .start_frame = videotoolbox_hevc_start_frame,
1148 .decode_slice = videotoolbox_hevc_decode_slice,
1149 .decode_params = videotoolbox_hevc_decode_params,
1150 .end_frame = videotoolbox_hevc_end_frame,
1151 .frame_params = videotoolbox_frame_params,
1152 .init = videotoolbox_common_init,
1158 .
name =
"h264_videotoolbox",
1166 .end_frame = videotoolbox_h264_end_frame,
1167 .frame_params = videotoolbox_frame_params,
1168 .init = videotoolbox_common_init,
1174 .
name =
"mpeg1_videotoolbox",
1179 .start_frame = videotoolbox_mpeg_start_frame,
1180 .decode_slice = videotoolbox_mpeg_decode_slice,
1181 .end_frame = videotoolbox_mpeg_end_frame,
1182 .frame_params = videotoolbox_frame_params,
1183 .init = videotoolbox_common_init,
1189 .
name =
"mpeg2_videotoolbox",
1194 .start_frame = videotoolbox_mpeg_start_frame,
1195 .decode_slice = videotoolbox_mpeg_decode_slice,
1196 .end_frame = videotoolbox_mpeg_end_frame,
1197 .frame_params = videotoolbox_frame_params,
1198 .init = videotoolbox_common_init,
1204 .
name =
"mpeg4_videotoolbox",
1209 .start_frame = videotoolbox_mpeg_start_frame,
1210 .decode_slice = videotoolbox_mpeg_decode_slice,
1211 .end_frame = videotoolbox_mpeg_end_frame,
1212 .frame_params = videotoolbox_frame_params,
1213 .init = videotoolbox_common_init,
1227 if (cv_pix_fmt_type == 0) {
1228 cv_pix_fmt_type = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
1238 return av_videotoolbox_alloc_context_with_pix_fmt(
AV_PIX_FMT_NONE,
false);
1250 avctx->
hwaccel_context = vtctx ?: av_videotoolbox_alloc_context_with_pix_fmt(pix_fmt, full_range);
1253 return videotoolbox_start(avctx);
1259 videotoolbox_stop(avctx);
int min_spatial_segmentation_idc
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
static enum AVPixelFormat pix_fmt
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it...
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
This structure describes decoded (raw) audio or video data.
ptrdiff_t const GLvoid * data
int coded_width
Bitstream width / height, may be different from width/height e.g.
int cm_codec_type
CoreMedia codec type that Videotoolbox will use to create the decompression session.
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
hardware decoding through Videotoolbox
enum AVColorRange color_range
MPEG vs JPEG YUV range.
static av_always_inline void bytestream2_init_writer(PutByteContext *p, uint8_t *buf, int buf_size)
int width
The allocated dimensions of the frames in this pool.
enum AVMediaType codec_type
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Convenience header that includes libavutil's core.
int is_avc
Used to parse AVC variant of H.264.
enum AVPixelFormat format
The pixel format identifying the underlying HW surface type.
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
const AVHWAccel ff_h264_videotoolbox_hwaccel
uint8_t entropy_coding_sync_enabled_flag
AVBufferRef * hw_frames_ctx
For hwaccel-format frames, this should be a reference to the AVHWFramesContext describing the frame...
#define av_assert0(cond)
assert() equivalent, that is always enabled.
uint8_t profile_compatibility_flag[32]
AVComponentDescriptor comp[4]
Parameters that describe how pixels are packed.
int ff_attach_decode_data(AVFrame *frame)
void * hwaccel_context
Hardware accelerator context.
uint32_t av_map_videotoolbox_format_from_pixfmt2(enum AVPixelFormat pix_fmt, bool full_range)
Same as av_map_videotoolbox_format_from_pixfmt function, but can map and return full range pixel form...
const char * av_color_range_name(enum AVColorRange range)
AVBufferRef * private_ref
AVBufferRef for internal use by a single libav* library.
An API-specific header for AV_HWDEVICE_TYPE_VIDEOTOOLBOX.
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
#define AV_LOG_VERBOSE
Detailed information.
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
uint8_t frame_only_constraint_flag
const AVHWAccel ff_mpeg2_videotoolbox_hwaccel
uint8_t temporal_id_nesting_flag
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
VTDecompressionSessionRef session
Videotoolbox decompression session object.
int av_hwframe_ctx_init(AVBufferRef *ref)
Finalize the context before use.
uint8_t tiles_enabled_flag
const AVHWAccel ff_hevc_videotoolbox_hwaccel
const char * name
Name of the hardware accelerated codec.
int width
picture width / height.
AVBufferRef * hw_frames_ctx
A reference to the AVHWFramesContext describing the input (for encoding) or output (decoding) frames...
Picture * current_picture_ptr
pointer to the current picture
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
H.264 / AVC / MPEG-4 part10 codec.
void * av_fast_realloc(void *ptr, unsigned int *size, size_t min_size)
Reallocate the given buffer if it is not large enough, otherwise do nothing.
static av_always_inline unsigned int bytestream2_put_buffer(PutByteContext *p, const uint8_t *src, unsigned int size)
preferred ID for MPEG-1/2 video decoding
int(* post_process)(void *logctx, AVFrame *frame)
The callback to perform some delayed processing on the frame right before it is returned to the calle...
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
#define bytestream2_put_ne24
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
main external API structure.
uint8_t * data
The data buffer.
CMVideoFormatDescriptionRef cm_fmt_desc
CoreMedia Format Description that Videotoolbox will use to create the decompression session...
static int FUNC() pps(CodedBitstreamContext *ctx, RWContext *rw, H264RawPPS *current)
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
This struct describes a set or pool of "hardware" frames (i.e.
H264Picture * cur_pic_ptr
static int FUNC() sps(CodedBitstreamContext *ctx, RWContext *rw, H264RawSPS *current)
OSType cv_pix_fmt_type
CVPixelBuffer Format Type that Videotoolbox will use for decoded frames.
const AVHWAccel ff_mpeg1_videotoolbox_hwaccel
static int FUNC() vps(CodedBitstreamContext *ctx, RWContext *rw, H265RawVPS *current)
void av_frame_unref(AVFrame *frame)
Unreference all the buffers referenced by frame and reset the frame fields.
#define flags(name, subs,...)
const AVHWAccel ff_mpeg4_videotoolbox_hwaccel
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
AVBufferRef * device_ref
A reference to the parent AVHWDeviceContext.
AVBufferRef * hw_frames_ctx
A reference to a data buffer.
This struct stores per-frame lavc-internal data and is attached to it via private_ref.
VTDecompressionOutputCallback output_callback
The output callback that must be passed to the session.
static int ref[MAX_W *MAX_W]
AVBufferRef * av_hwframe_ctx_alloc(AVBufferRef *device_ref_in)
Allocate an AVHWFramesContext tied to a given device context.
void * hwaccel_priv_data
hwaccel-specific private data
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
struct AVVideotoolboxContext * vt_ctx
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
enum AVPixelFormat av_map_videotoolbox_format_to_pixfmt(uint32_t cv_fmt)
Convert a VideoToolbox (actually CoreVideo) format to AVPixelFormat.
struct AVCodecInternal * internal
Private context used for internal data.
const AVHWAccel ff_h263_videotoolbox_hwaccel
struct AVBufferRef * cached_hw_frames_ctx
#define bytestream2_put_ne32
uint8_t progressive_source_flag
#define bytestream2_put_ne16
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
int depth
Number of bits in the component.
AVBufferRef * hw_device_ctx
A reference to the AVHWDeviceContext describing the device which will be used by a hardware encoder/d...
uint8_t non_packed_constraint_flag
enum AVPixelFormat sw_format
The pixel format identifying the actual data layout of the hardware frames.
#define AVERROR_EXTERNAL
Generic error in an external library.
AVPixelFormat
Pixel format.
uint8_t interlaced_source_flag
This struct holds all the information that needs to be passed between the caller and libavcodec for i...
static av_always_inline int bytestream2_size_p(PutByteContext *p)