FFmpeg
videotoolboxenc.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2015 Rick Kern <kernrj@gmail.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <VideoToolbox/VideoToolbox.h>
22 #include <CoreVideo/CoreVideo.h>
23 #include <CoreMedia/CoreMedia.h>
24 #include <TargetConditionals.h>
25 #include <Availability.h>
26 #include "avcodec.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/avassert.h"
29 #include "libavutil/avstring.h"
30 #include "libavcodec/avcodec.h"
31 #include "libavutil/pixdesc.h"
32 #include "internal.h"
33 #include <pthread.h>
34 #include "atsc_a53.h"
35 #include "h264.h"
36 #include "h264_sei.h"
37 #include <dlfcn.h>
38 
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
41 #endif
42 
43 #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
44 enum { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange = 'xf20' };
45 enum { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange = 'x420' };
46 #endif
47 
48 typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc,
49  size_t parameterSetIndex,
50  const uint8_t **parameterSetPointerOut,
51  size_t *parameterSetSizeOut,
52  size_t *parameterSetCountOut,
53  int *NALUnitHeaderLengthOut);
54 
55 //These symbols may not be present
56 static struct{
57  CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020;
58  CFStringRef kCVImageBufferTransferFunction_ITU_R_2020;
59  CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020;
60 
61  CFStringRef kVTCompressionPropertyKey_H264EntropyMode;
62  CFStringRef kVTH264EntropyMode_CAVLC;
63  CFStringRef kVTH264EntropyMode_CABAC;
64 
65  CFStringRef kVTProfileLevel_H264_Baseline_4_0;
66  CFStringRef kVTProfileLevel_H264_Baseline_4_2;
67  CFStringRef kVTProfileLevel_H264_Baseline_5_0;
68  CFStringRef kVTProfileLevel_H264_Baseline_5_1;
69  CFStringRef kVTProfileLevel_H264_Baseline_5_2;
70  CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel;
71  CFStringRef kVTProfileLevel_H264_Main_4_2;
72  CFStringRef kVTProfileLevel_H264_Main_5_1;
73  CFStringRef kVTProfileLevel_H264_Main_5_2;
74  CFStringRef kVTProfileLevel_H264_Main_AutoLevel;
75  CFStringRef kVTProfileLevel_H264_High_3_0;
76  CFStringRef kVTProfileLevel_H264_High_3_1;
77  CFStringRef kVTProfileLevel_H264_High_3_2;
78  CFStringRef kVTProfileLevel_H264_High_4_0;
79  CFStringRef kVTProfileLevel_H264_High_4_1;
80  CFStringRef kVTProfileLevel_H264_High_4_2;
81  CFStringRef kVTProfileLevel_H264_High_5_1;
82  CFStringRef kVTProfileLevel_H264_High_5_2;
83  CFStringRef kVTProfileLevel_H264_High_AutoLevel;
84  CFStringRef kVTProfileLevel_H264_Extended_5_0;
85  CFStringRef kVTProfileLevel_H264_Extended_AutoLevel;
86 
87  CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel;
88  CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel;
89 
90  CFStringRef kVTCompressionPropertyKey_RealTime;
91 
92  CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder;
93  CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder;
94 
95  getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
96 } compat_keys;
97 
98 #define GET_SYM(symbol, defaultVal) \
99 do{ \
100  CFStringRef* handle = (CFStringRef*)dlsym(RTLD_DEFAULT, #symbol); \
101  if(!handle) \
102  compat_keys.symbol = CFSTR(defaultVal); \
103  else \
104  compat_keys.symbol = *handle; \
105 }while(0)
106 
107 static pthread_once_t once_ctrl = PTHREAD_ONCE_INIT;
108 
109 static void loadVTEncSymbols(){
110  compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex =
111  (getParameterSetAtIndex)dlsym(
112  RTLD_DEFAULT,
113  "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex"
114  );
115 
116  GET_SYM(kCVImageBufferColorPrimaries_ITU_R_2020, "ITU_R_2020");
117  GET_SYM(kCVImageBufferTransferFunction_ITU_R_2020, "ITU_R_2020");
118  GET_SYM(kCVImageBufferYCbCrMatrix_ITU_R_2020, "ITU_R_2020");
119 
120  GET_SYM(kVTCompressionPropertyKey_H264EntropyMode, "H264EntropyMode");
121  GET_SYM(kVTH264EntropyMode_CAVLC, "CAVLC");
122  GET_SYM(kVTH264EntropyMode_CABAC, "CABAC");
123 
124  GET_SYM(kVTProfileLevel_H264_Baseline_4_0, "H264_Baseline_4_0");
125  GET_SYM(kVTProfileLevel_H264_Baseline_4_2, "H264_Baseline_4_2");
126  GET_SYM(kVTProfileLevel_H264_Baseline_5_0, "H264_Baseline_5_0");
127  GET_SYM(kVTProfileLevel_H264_Baseline_5_1, "H264_Baseline_5_1");
128  GET_SYM(kVTProfileLevel_H264_Baseline_5_2, "H264_Baseline_5_2");
129  GET_SYM(kVTProfileLevel_H264_Baseline_AutoLevel, "H264_Baseline_AutoLevel");
130  GET_SYM(kVTProfileLevel_H264_Main_4_2, "H264_Main_4_2");
131  GET_SYM(kVTProfileLevel_H264_Main_5_1, "H264_Main_5_1");
132  GET_SYM(kVTProfileLevel_H264_Main_5_2, "H264_Main_5_2");
133  GET_SYM(kVTProfileLevel_H264_Main_AutoLevel, "H264_Main_AutoLevel");
134  GET_SYM(kVTProfileLevel_H264_High_3_0, "H264_High_3_0");
135  GET_SYM(kVTProfileLevel_H264_High_3_1, "H264_High_3_1");
136  GET_SYM(kVTProfileLevel_H264_High_3_2, "H264_High_3_2");
137  GET_SYM(kVTProfileLevel_H264_High_4_0, "H264_High_4_0");
138  GET_SYM(kVTProfileLevel_H264_High_4_1, "H264_High_4_1");
139  GET_SYM(kVTProfileLevel_H264_High_4_2, "H264_High_4_2");
140  GET_SYM(kVTProfileLevel_H264_High_5_1, "H264_High_5_1");
141  GET_SYM(kVTProfileLevel_H264_High_5_2, "H264_High_5_2");
142  GET_SYM(kVTProfileLevel_H264_High_AutoLevel, "H264_High_AutoLevel");
143  GET_SYM(kVTProfileLevel_H264_Extended_5_0, "H264_Extended_5_0");
144  GET_SYM(kVTProfileLevel_H264_Extended_AutoLevel, "H264_Extended_AutoLevel");
145 
146  GET_SYM(kVTProfileLevel_HEVC_Main_AutoLevel, "HEVC_Main_AutoLevel");
147  GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel, "HEVC_Main10_AutoLevel");
148 
149  GET_SYM(kVTCompressionPropertyKey_RealTime, "RealTime");
150 
151  GET_SYM(kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
152  "EnableHardwareAcceleratedVideoEncoder");
153  GET_SYM(kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
154  "RequireHardwareAcceleratedVideoEncoder");
155 }
156 
157 typedef enum VT_H264Profile {
158  H264_PROF_AUTO,
159  H264_PROF_BASELINE,
160  H264_PROF_MAIN,
161  H264_PROF_HIGH,
162  H264_PROF_EXTENDED,
163  H264_PROF_COUNT
164 } VT_H264Profile;
165 
166 typedef enum VTH264Entropy{
167  VT_ENTROPY_NOT_SET,
168  VT_CAVLC,
169  VT_CABAC
170 } VTH264Entropy;
171 
172 typedef enum VT_HEVCProfile {
173  HEVC_PROF_AUTO,
174  HEVC_PROF_MAIN,
175  HEVC_PROF_MAIN10,
176  HEVC_PROF_COUNT
177 } VT_HEVCProfile;
178 
179 static const uint8_t start_code[] = { 0, 0, 0, 1 };
180 
181 typedef struct ExtraSEI {
182  void *data;
183  size_t size;
184 } ExtraSEI;
185 
186 typedef struct BufNode {
187  CMSampleBufferRef cm_buffer;
188  ExtraSEI *sei;
189  struct BufNode* next;
190  int error;
191 } BufNode;
192 
193 typedef struct VTEncContext {
194  AVClass *class;
195  enum AVCodecID codec_id;
196  VTCompressionSessionRef session;
197  CFStringRef ycbcr_matrix;
198  CFStringRef color_primaries;
199  CFStringRef transfer_function;
200  getParameterSetAtIndex get_param_set_func;
201 
202  pthread_mutex_t lock;
203  pthread_cond_t cv_sample_sent;
204 
205  int async_error;
206 
207  BufNode *q_head;
208  BufNode *q_tail;
209 
210  int64_t frame_ct_out;
211  int64_t frame_ct_in;
212 
213  int64_t first_pts;
214  int64_t dts_delta;
215 
216  int64_t profile;
217  int64_t level;
218  int64_t entropy;
219  int64_t realtime;
220  int64_t frames_before;
221  int64_t frames_after;
222 
223  int64_t allow_sw;
224  int64_t require_sw;
225 
226  bool flushing;
227  bool has_b_frames;
228  bool warned_color_range;
229 
230  /* can't be bool type since AVOption will access it as int */
231  int a53_cc;
232 } VTEncContext;
233 
234 static int vtenc_populate_extradata(AVCodecContext *avctx,
235  CMVideoCodecType codec_type,
236  CFStringRef profile_level,
237  CFNumberRef gamma_level,
238  CFDictionaryRef enc_info,
239  CFDictionaryRef pixel_buffer_info);
240 
241 /**
242  * NULL-safe release of *refPtr, and sets value to NULL.
243  */
244 static void vt_release_num(CFNumberRef* refPtr){
245  if (!*refPtr) {
246  return;
247  }
248 
249  CFRelease(*refPtr);
250  *refPtr = NULL;
251 }
252 
253 static void set_async_error(VTEncContext *vtctx, int err)
254 {
255  BufNode *info;
256 
257  pthread_mutex_lock(&vtctx->lock);
258 
259  vtctx->async_error = err;
260 
261  info = vtctx->q_head;
262  vtctx->q_head = vtctx->q_tail = NULL;
263 
264  while (info) {
265  BufNode *next = info->next;
266  CFRelease(info->cm_buffer);
267  av_free(info);
268  info = next;
269  }
270 
271  pthread_mutex_unlock(&vtctx->lock);
272 }
273 
274 static void clear_frame_queue(VTEncContext *vtctx)
275 {
276  set_async_error(vtctx, 0);
277 }
278 
279 static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
280 {
281  BufNode *info;
282 
283  pthread_mutex_lock(&vtctx->lock);
284 
285  if (vtctx->async_error) {
286  pthread_mutex_unlock(&vtctx->lock);
287  return vtctx->async_error;
288  }
289 
290  if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) {
291  *buf = NULL;
292 
293  pthread_mutex_unlock(&vtctx->lock);
294  return 0;
295  }
296 
297  while (!vtctx->q_head && !vtctx->async_error && wait && !vtctx->flushing) {
298  pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
299  }
300 
301  if (!vtctx->q_head) {
302  pthread_mutex_unlock(&vtctx->lock);
303  *buf = NULL;
304  return 0;
305  }
306 
307  info = vtctx->q_head;
308  vtctx->q_head = vtctx->q_head->next;
309  if (!vtctx->q_head) {
310  vtctx->q_tail = NULL;
311  }
312 
313  vtctx->frame_ct_out++;
314  pthread_mutex_unlock(&vtctx->lock);
315 
316  *buf = info->cm_buffer;
317  if (sei && *buf) {
318  *sei = info->sei;
319  } else if (info->sei) {
320  if (info->sei->data) av_free(info->sei->data);
321  av_free(info->sei);
322  }
323  av_free(info);
324 
325 
326  return 0;
327 }
328 
329 static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
330 {
331  BufNode *info = av_malloc(sizeof(BufNode));
332  if (!info) {
333  set_async_error(vtctx, AVERROR(ENOMEM));
334  return;
335  }
336 
337  CFRetain(buffer);
338  info->cm_buffer = buffer;
339  info->sei = sei;
340  info->next = NULL;
341 
342  pthread_mutex_lock(&vtctx->lock);
343 
344  if (!vtctx->q_head) {
345  vtctx->q_head = info;
346  } else {
347  vtctx->q_tail->next = info;
348  }
349 
350  vtctx->q_tail = info;
351 
352  pthread_cond_signal(&vtctx->cv_sample_sent);
353  pthread_mutex_unlock(&vtctx->lock);
354 }
355 
356 static int count_nalus(size_t length_code_size,
357  CMSampleBufferRef sample_buffer,
358  int *count)
359 {
360  size_t offset = 0;
361  int status;
362  int nalu_ct = 0;
363  uint8_t size_buf[4];
364  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
365  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
366 
367  if (length_code_size > 4)
368  return AVERROR_INVALIDDATA;
369 
370  while (offset < src_size) {
371  size_t curr_src_len;
372  size_t box_len = 0;
373  size_t i;
374 
375  status = CMBlockBufferCopyDataBytes(block,
376  offset,
377  length_code_size,
378  size_buf);
379 
380  for (i = 0; i < length_code_size; i++) {
381  box_len <<= 8;
382  box_len |= size_buf[i];
383  }
384 
385  curr_src_len = box_len + length_code_size;
386  offset += curr_src_len;
387 
388  nalu_ct++;
389  }
390 
391  *count = nalu_ct;
392  return 0;
393 }
394 
395 static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
396 {
397  switch (id) {
398  case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
399  case AV_CODEC_ID_HEVC: return kCMVideoCodecType_HEVC;
400  default: return 0;
401  }
402 }
403 
404 /**
405  * Get the parameter sets from a CMSampleBufferRef.
406  * @param dst If *dst isn't NULL, the parameters are copied into existing
407  * memory. *dst_size must be set accordingly when *dst != NULL.
408  * If *dst is NULL, it will be allocated.
409  * In all cases, *dst_size is set to the number of bytes used starting
410  * at *dst.
411  */
412 static int get_params_size(
413  AVCodecContext *avctx,
414  CMVideoFormatDescriptionRef vid_fmt,
415  size_t *size)
416 {
417  VTEncContext *vtctx = avctx->priv_data;
418  size_t total_size = 0;
419  size_t ps_count;
420  int is_count_bad = 0;
421  size_t i;
422  int status;
423  status = vtctx->get_param_set_func(vid_fmt,
424  0,
425  NULL,
426  NULL,
427  &ps_count,
428  NULL);
429  if (status) {
430  is_count_bad = 1;
431  ps_count = 0;
432  status = 0;
433  }
434 
435  for (i = 0; i < ps_count || is_count_bad; i++) {
436  const uint8_t *ps;
437  size_t ps_size;
438  status = vtctx->get_param_set_func(vid_fmt,
439  i,
440  &ps,
441  &ps_size,
442  NULL,
443  NULL);
444  if (status) {
445  /*
446  * When ps_count is invalid, status != 0 ends the loop normally
447  * unless we didn't get any parameter sets.
448  */
449  if (i > 0 && is_count_bad) status = 0;
450 
451  break;
452  }
453 
454  total_size += ps_size + sizeof(start_code);
455  }
456 
457  if (status) {
458  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status);
459  return AVERROR_EXTERNAL;
460  }
461 
462  *size = total_size;
463  return 0;
464 }
465 
466 static int copy_param_sets(
467  AVCodecContext *avctx,
468  CMVideoFormatDescriptionRef vid_fmt,
469  uint8_t *dst,
470  size_t dst_size)
471 {
472  VTEncContext *vtctx = avctx->priv_data;
473  size_t ps_count;
474  int is_count_bad = 0;
475  int status;
476  size_t offset = 0;
477  size_t i;
478 
479  status = vtctx->get_param_set_func(vid_fmt,
480  0,
481  NULL,
482  NULL,
483  &ps_count,
484  NULL);
485  if (status) {
486  is_count_bad = 1;
487  ps_count = 0;
488  status = 0;
489  }
490 
491 
492  for (i = 0; i < ps_count || is_count_bad; i++) {
493  const uint8_t *ps;
494  size_t ps_size;
495  size_t next_offset;
496 
497  status = vtctx->get_param_set_func(vid_fmt,
498  i,
499  &ps,
500  &ps_size,
501  NULL,
502  NULL);
503  if (status) {
504  if (i > 0 && is_count_bad) status = 0;
505 
506  break;
507  }
508 
509  next_offset = offset + sizeof(start_code) + ps_size;
510  if (dst_size < next_offset) {
511  av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
512  return AVERROR_BUFFER_TOO_SMALL;
513  }
514 
515  memcpy(dst + offset, start_code, sizeof(start_code));
516  offset += sizeof(start_code);
517 
518  memcpy(dst + offset, ps, ps_size);
519  offset = next_offset;
520  }
521 
522  if (status) {
523  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status);
524  return AVERROR_EXTERNAL;
525  }
526 
527  return 0;
528 }
529 
530 static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
531 {
532  CMVideoFormatDescriptionRef vid_fmt;
533  size_t total_size;
534  int status;
535 
536  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
537  if (!vid_fmt) {
538  av_log(avctx, AV_LOG_ERROR, "No video format.\n");
539  return AVERROR_EXTERNAL;
540  }
541 
542  status = get_params_size(avctx, vid_fmt, &total_size);
543  if (status) {
544  av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
545  return status;
546  }
547 
548  avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE);
549  if (!avctx->extradata) {
550  return AVERROR(ENOMEM);
551  }
552  avctx->extradata_size = total_size;
553 
554  status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
555 
556  if (status) {
557  av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
558  return status;
559  }
560 
561  return 0;
562 }
563 
564 static void vtenc_output_callback(
565  void *ctx,
566  void *sourceFrameCtx,
567  OSStatus status,
568  VTEncodeInfoFlags flags,
569  CMSampleBufferRef sample_buffer)
570 {
571  AVCodecContext *avctx = ctx;
572  VTEncContext *vtctx = avctx->priv_data;
573  ExtraSEI *sei = sourceFrameCtx;
574 
575  if (vtctx->async_error) {
576  return;
577  }
578 
579  if (status) {
580  av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status);
581  set_async_error(vtctx, AVERROR_EXTERNAL);
582  return;
583  }
584 
585  if (!sample_buffer) {
586  return;
587  }
588 
589  if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
590  int set_status = set_extradata(avctx, sample_buffer);
591  if (set_status) {
592  set_async_error(vtctx, set_status);
593  return;
594  }
595  }
596 
597  vtenc_q_push(vtctx, sample_buffer, sei);
598 }
599 
600 static int get_length_code_size(
601  AVCodecContext *avctx,
602  CMSampleBufferRef sample_buffer,
603  size_t *size)
604 {
605  VTEncContext *vtctx = avctx->priv_data;
606  CMVideoFormatDescriptionRef vid_fmt;
607  int isize;
608  int status;
609 
610  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
611  if (!vid_fmt) {
612  av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
613  return AVERROR_EXTERNAL;
614  }
615 
616  status = vtctx->get_param_set_func(vid_fmt,
617  0,
618  NULL,
619  NULL,
620  NULL,
621  &isize);
622  if (status) {
623  av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
624  return AVERROR_EXTERNAL;
625  }
626 
627  *size = isize;
628  return 0;
629 }
630 
631 /*
632  * Returns true on success.
633  *
634  * If profile_level_val is NULL and this method returns true, don't specify the
635  * profile/level to the encoder.
636  */
637 static bool get_vt_h264_profile_level(AVCodecContext *avctx,
638  CFStringRef *profile_level_val)
639 {
640  VTEncContext *vtctx = avctx->priv_data;
641  int64_t profile = vtctx->profile;
642 
643  if (profile == H264_PROF_AUTO && vtctx->level) {
644  //Need to pick a profile if level is not auto-selected.
645  profile = vtctx->has_b_frames ? H264_PROF_MAIN : H264_PROF_BASELINE;
646  }
647 
648  *profile_level_val = NULL;
649 
650  switch (profile) {
651  case H264_PROF_AUTO:
652  return true;
653 
654  case H264_PROF_BASELINE:
655  switch (vtctx->level) {
656  case 0: *profile_level_val =
657  compat_keys.kVTProfileLevel_H264_Baseline_AutoLevel; break;
658  case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3; break;
659  case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0; break;
660  case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1; break;
661  case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2; break;
662  case 40: *profile_level_val =
663  compat_keys.kVTProfileLevel_H264_Baseline_4_0; break;
664  case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1; break;
665  case 42: *profile_level_val =
666  compat_keys.kVTProfileLevel_H264_Baseline_4_2; break;
667  case 50: *profile_level_val =
668  compat_keys.kVTProfileLevel_H264_Baseline_5_0; break;
669  case 51: *profile_level_val =
670  compat_keys.kVTProfileLevel_H264_Baseline_5_1; break;
671  case 52: *profile_level_val =
672  compat_keys.kVTProfileLevel_H264_Baseline_5_2; break;
673  }
674  break;
675 
676  case H264_PROF_MAIN:
677  switch (vtctx->level) {
678  case 0: *profile_level_val =
679  compat_keys.kVTProfileLevel_H264_Main_AutoLevel; break;
680  case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0; break;
681  case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1; break;
682  case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2; break;
683  case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0; break;
684  case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1; break;
685  case 42: *profile_level_val =
686  compat_keys.kVTProfileLevel_H264_Main_4_2; break;
687  case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0; break;
688  case 51: *profile_level_val =
689  compat_keys.kVTProfileLevel_H264_Main_5_1; break;
690  case 52: *profile_level_val =
691  compat_keys.kVTProfileLevel_H264_Main_5_2; break;
692  }
693  break;
694 
695  case H264_PROF_HIGH:
696  switch (vtctx->level) {
697  case 0: *profile_level_val =
698  compat_keys.kVTProfileLevel_H264_High_AutoLevel; break;
699  case 30: *profile_level_val =
700  compat_keys.kVTProfileLevel_H264_High_3_0; break;
701  case 31: *profile_level_val =
702  compat_keys.kVTProfileLevel_H264_High_3_1; break;
703  case 32: *profile_level_val =
704  compat_keys.kVTProfileLevel_H264_High_3_2; break;
705  case 40: *profile_level_val =
706  compat_keys.kVTProfileLevel_H264_High_4_0; break;
707  case 41: *profile_level_val =
708  compat_keys.kVTProfileLevel_H264_High_4_1; break;
709  case 42: *profile_level_val =
710  compat_keys.kVTProfileLevel_H264_High_4_2; break;
711  case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0; break;
712  case 51: *profile_level_val =
713  compat_keys.kVTProfileLevel_H264_High_5_1; break;
714  case 52: *profile_level_val =
715  compat_keys.kVTProfileLevel_H264_High_5_2; break;
716  }
717  break;
718  case H264_PROF_EXTENDED:
719  switch (vtctx->level) {
720  case 0: *profile_level_val =
721  compat_keys.kVTProfileLevel_H264_Extended_AutoLevel; break;
722  case 50: *profile_level_val =
723  compat_keys.kVTProfileLevel_H264_Extended_5_0; break;
724  }
725  break;
726  }
727 
728  if (!*profile_level_val) {
729  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
730  return false;
731  }
732 
733  return true;
734 }
735 
736 /*
737  * Returns true on success.
738  *
739  * If profile_level_val is NULL and this method returns true, don't specify the
740  * profile/level to the encoder.
741  */
742 static bool get_vt_hevc_profile_level(AVCodecContext *avctx,
743  CFStringRef *profile_level_val)
744 {
745  VTEncContext *vtctx = avctx->priv_data;
746  int64_t profile = vtctx->profile;
747 
748  *profile_level_val = NULL;
749 
750  switch (profile) {
751  case HEVC_PROF_AUTO:
752  return true;
753  case HEVC_PROF_MAIN:
754  *profile_level_val =
755  compat_keys.kVTProfileLevel_HEVC_Main_AutoLevel;
756  break;
757  case HEVC_PROF_MAIN10:
758  *profile_level_val =
759  compat_keys.kVTProfileLevel_HEVC_Main10_AutoLevel;
760  break;
761  }
762 
763  if (!*profile_level_val) {
764  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
765  return false;
766  }
767 
768  return true;
769 }
770 
771 static int get_cv_pixel_format(AVCodecContext* avctx,
772  enum AVPixelFormat fmt,
773  enum AVColorRange range,
774  int* av_pixel_format,
775  int* range_guessed)
776 {
777  if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG &&
778  range != AVCOL_RANGE_JPEG;
779 
780  //MPEG range is used when no range is set
781  if (fmt == AV_PIX_FMT_NV12) {
782  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
783  kCVPixelFormatType_420YpCbCr8BiPlanarFullRange :
784  kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
785  } else if (fmt == AV_PIX_FMT_YUV420P) {
786  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
787  kCVPixelFormatType_420YpCbCr8PlanarFullRange :
788  kCVPixelFormatType_420YpCbCr8Planar;
789  } else if (fmt == AV_PIX_FMT_P010LE) {
790  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
791  kCVPixelFormatType_420YpCbCr10BiPlanarFullRange :
792  kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange;
793  } else {
794  return AVERROR(EINVAL);
795  }
796 
797  return 0;
798 }
799 
800 static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) {
801  VTEncContext *vtctx = avctx->priv_data;
802 
803  if (vtctx->color_primaries) {
804  CFDictionarySetValue(dict,
805  kCVImageBufferColorPrimariesKey,
806  vtctx->color_primaries);
807  }
808 
809  if (vtctx->transfer_function) {
810  CFDictionarySetValue(dict,
811  kCVImageBufferTransferFunctionKey,
812  vtctx->transfer_function);
813  }
814 
815  if (vtctx->ycbcr_matrix) {
816  CFDictionarySetValue(dict,
817  kCVImageBufferYCbCrMatrixKey,
818  vtctx->ycbcr_matrix);
819  }
820 }
821 
822 static int create_cv_pixel_buffer_info(AVCodecContext* avctx,
823  CFMutableDictionaryRef* dict)
824 {
825  CFNumberRef cv_color_format_num = NULL;
826  CFNumberRef width_num = NULL;
827  CFNumberRef height_num = NULL;
828  CFMutableDictionaryRef pixel_buffer_info = NULL;
829  int cv_color_format;
830  int status = get_cv_pixel_format(avctx,
831  avctx->pix_fmt,
832  avctx->color_range,
833  &cv_color_format,
834  NULL);
835  if (status) return status;
836 
837  pixel_buffer_info = CFDictionaryCreateMutable(
838  kCFAllocatorDefault,
839  20,
840  &kCFCopyStringDictionaryKeyCallBacks,
841  &kCFTypeDictionaryValueCallBacks);
842 
843  if (!pixel_buffer_info) goto pbinfo_nomem;
844 
845  cv_color_format_num = CFNumberCreate(kCFAllocatorDefault,
846  kCFNumberSInt32Type,
847  &cv_color_format);
848  if (!cv_color_format_num) goto pbinfo_nomem;
849 
850  CFDictionarySetValue(pixel_buffer_info,
851  kCVPixelBufferPixelFormatTypeKey,
852  cv_color_format_num);
853  vt_release_num(&cv_color_format_num);
854 
855  width_num = CFNumberCreate(kCFAllocatorDefault,
856  kCFNumberSInt32Type,
857  &avctx->width);
858  if (!width_num) return AVERROR(ENOMEM);
859 
860  CFDictionarySetValue(pixel_buffer_info,
861  kCVPixelBufferWidthKey,
862  width_num);
863  vt_release_num(&width_num);
864 
865  height_num = CFNumberCreate(kCFAllocatorDefault,
866  kCFNumberSInt32Type,
867  &avctx->height);
868  if (!height_num) goto pbinfo_nomem;
869 
870  CFDictionarySetValue(pixel_buffer_info,
871  kCVPixelBufferHeightKey,
872  height_num);
873  vt_release_num(&height_num);
874 
875  add_color_attr(avctx, pixel_buffer_info);
876 
877  *dict = pixel_buffer_info;
878  return 0;
879 
880 pbinfo_nomem:
881  vt_release_num(&cv_color_format_num);
882  vt_release_num(&width_num);
883  vt_release_num(&height_num);
884  if (pixel_buffer_info) CFRelease(pixel_buffer_info);
885 
886  return AVERROR(ENOMEM);
887 }
888 
889 static int get_cv_color_primaries(AVCodecContext *avctx,
890  CFStringRef *primaries)
891 {
892  enum AVColorPrimaries pri = avctx->color_primaries;
893  switch (pri) {
894  case AVCOL_PRI_UNSPECIFIED:
895  *primaries = NULL;
896  break;
897 
898  case AVCOL_PRI_BT470BG:
899  *primaries = kCVImageBufferColorPrimaries_EBU_3213;
900  break;
901 
902  case AVCOL_PRI_SMPTE170M:
903  *primaries = kCVImageBufferColorPrimaries_SMPTE_C;
904  break;
905 
906  case AVCOL_PRI_BT709:
907  *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
908  break;
909 
910  case AVCOL_PRI_BT2020:
911  *primaries = compat_keys.kCVImageBufferColorPrimaries_ITU_R_2020;
912  break;
913 
914  default:
915  av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri));
916  *primaries = NULL;
917  return -1;
918  }
919 
920  return 0;
921 }
922 
923 static int get_cv_transfer_function(AVCodecContext *avctx,
924  CFStringRef *transfer_fnc,
925  CFNumberRef *gamma_level)
926 {
927  enum AVColorTransferCharacteristic trc = avctx->color_trc;
928  Float32 gamma;
929  *gamma_level = NULL;
930 
931  switch (trc) {
932  case AVCOL_TRC_UNSPECIFIED:
933  *transfer_fnc = NULL;
934  break;
935 
936  case AVCOL_TRC_BT709:
937  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2;
938  break;
939 
940  case AVCOL_TRC_SMPTE240M:
941  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
942  break;
943 
944 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ
945  case AVCOL_TRC_SMPTE2084:
946  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
947  break;
948 #endif
949 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_LINEAR
950  case AVCOL_TRC_LINEAR:
951  *transfer_fnc = kCVImageBufferTransferFunction_Linear;
952  break;
953 #endif
954 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
955  case AVCOL_TRC_ARIB_STD_B67:
956  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
957  break;
958 #endif
959 
960  case AVCOL_TRC_GAMMA22:
961  gamma = 2.2;
962  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
963  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
964  break;
965 
966  case AVCOL_TRC_GAMMA28:
967  gamma = 2.8;
968  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
969  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
970  break;
971 
972  case AVCOL_TRC_BT2020_10:
973  case AVCOL_TRC_BT2020_12:
974  *transfer_fnc = compat_keys.kCVImageBufferTransferFunction_ITU_R_2020;
975  break;
976 
977  default:
978  *transfer_fnc = NULL;
979  av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc));
980  return -1;
981  }
982 
983  return 0;
984 }
985 
986 static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) {
987  switch(avctx->colorspace) {
988  case AVCOL_SPC_BT709:
989  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
990  break;
991 
992  case AVCOL_SPC_UNSPECIFIED:
993  *matrix = NULL;
994  break;
995 
996  case AVCOL_SPC_BT470BG:
997  case AVCOL_SPC_SMPTE170M:
998  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
999  break;
1000 
1001  case AVCOL_SPC_SMPTE240M:
1002  *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
1003  break;
1004 
1005  case AVCOL_SPC_BT2020_NCL:
1006  *matrix = compat_keys.kCVImageBufferYCbCrMatrix_ITU_R_2020;
1007  break;
1008 
1009  default:
1010  av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace));
1011  return -1;
1012  }
1013 
1014  return 0;
1015 }
1016 
1017 static int vtenc_create_encoder(AVCodecContext *avctx,
1018  CMVideoCodecType codec_type,
1019  CFStringRef profile_level,
1020  CFNumberRef gamma_level,
1021  CFDictionaryRef enc_info,
1022  CFDictionaryRef pixel_buffer_info,
1023  VTCompressionSessionRef *session)
1024 {
1025  VTEncContext *vtctx = avctx->priv_data;
1026  SInt32 bit_rate = avctx->bit_rate;
1027  SInt32 max_rate = avctx->rc_max_rate;
1028  CFNumberRef bit_rate_num;
1029  CFNumberRef bytes_per_second;
1030  CFNumberRef one_second;
1031  CFArrayRef data_rate_limits;
1032  int64_t bytes_per_second_value = 0;
1033  int64_t one_second_value = 0;
1034  void *nums[2];
1035 
1036  int status = VTCompressionSessionCreate(kCFAllocatorDefault,
1037  avctx->width,
1038  avctx->height,
1039  codec_type,
1040  enc_info,
1041  pixel_buffer_info,
1042  kCFAllocatorDefault,
1043  vtenc_output_callback,
1044  avctx,
1045  session);
1046 
1047  if (status || !vtctx->session) {
1048  av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status);
1049 
1050 #if !TARGET_OS_IPHONE
1051  if (!vtctx->allow_sw) {
1052  av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n");
1053  }
1054 #endif
1055 
1056  return AVERROR_EXTERNAL;
1057  }
1058 
1059  bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
1060  kCFNumberSInt32Type,
1061  &bit_rate);
1062  if (!bit_rate_num) return AVERROR(ENOMEM);
1063 
1064  status = VTSessionSetProperty(vtctx->session,
1065  kVTCompressionPropertyKey_AverageBitRate,
1066  bit_rate_num);
1067  CFRelease(bit_rate_num);
1068 
1069  if (status) {
1070  av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status);
1071  return AVERROR_EXTERNAL;
1072  }
1073 
1074  if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
1075  // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
1076  bytes_per_second_value = max_rate >> 3;
1077  bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
1078  kCFNumberSInt64Type,
1079  &bytes_per_second_value);
1080  if (!bytes_per_second) {
1081  return AVERROR(ENOMEM);
1082  }
1083  one_second_value = 1;
1084  one_second = CFNumberCreate(kCFAllocatorDefault,
1085  kCFNumberSInt64Type,
1086  &one_second_value);
1087  if (!one_second) {
1088  CFRelease(bytes_per_second);
1089  return AVERROR(ENOMEM);
1090  }
1091  nums[0] = (void *)bytes_per_second;
1092  nums[1] = (void *)one_second;
1093  data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
1094  (const void **)nums,
1095  2,
1096  &kCFTypeArrayCallBacks);
1097 
1098  if (!data_rate_limits) {
1099  CFRelease(bytes_per_second);
1100  CFRelease(one_second);
1101  return AVERROR(ENOMEM);
1102  }
1103  status = VTSessionSetProperty(vtctx->session,
1104  kVTCompressionPropertyKey_DataRateLimits,
1105  data_rate_limits);
1106 
1107  CFRelease(bytes_per_second);
1108  CFRelease(one_second);
1109  CFRelease(data_rate_limits);
1110 
1111  if (status) {
1112  av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
1113  return AVERROR_EXTERNAL;
1114  }
1115  }
1116 
1117  if (profile_level) {
1118  status = VTSessionSetProperty(vtctx->session,
1119  kVTCompressionPropertyKey_ProfileLevel,
1120  profile_level);
1121  if (status) {
1122  av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d. Output will be encoded using a supported profile/level combination.\n", status);
1123  }
1124  }
1125 
1126  if (avctx->gop_size > 0) {
1127  CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
1128  kCFNumberIntType,
1129  &avctx->gop_size);
1130  if (!interval) {
1131  return AVERROR(ENOMEM);
1132  }
1133 
1134  status = VTSessionSetProperty(vtctx->session,
1135  kVTCompressionPropertyKey_MaxKeyFrameInterval,
1136  interval);
1137  CFRelease(interval);
1138 
1139  if (status) {
1140  av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status);
1141  return AVERROR_EXTERNAL;
1142  }
1143  }
1144 
1145  if (vtctx->frames_before) {
1146  status = VTSessionSetProperty(vtctx->session,
1147  kVTCompressionPropertyKey_MoreFramesBeforeStart,
1148  kCFBooleanTrue);
1149 
1150  if (status == kVTPropertyNotSupportedErr) {
1151  av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n");
1152  } else if (status) {
1153  av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status);
1154  }
1155  }
1156 
1157  if (vtctx->frames_after) {
1158  status = VTSessionSetProperty(vtctx->session,
1159  kVTCompressionPropertyKey_MoreFramesAfterEnd,
1160  kCFBooleanTrue);
1161 
1162  if (status == kVTPropertyNotSupportedErr) {
1163  av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n");
1164  } else if (status) {
1165  av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status);
1166  }
1167  }
1168 
1169  if (avctx->sample_aspect_ratio.num != 0) {
1170  CFNumberRef num;
1171  CFNumberRef den;
1172  CFMutableDictionaryRef par;
1173  AVRational *avpar = &avctx->sample_aspect_ratio;
1174 
1175  av_reduce(&avpar->num, &avpar->den,
1176  avpar->num, avpar->den,
1177  0xFFFFFFFF);
1178 
1179  num = CFNumberCreate(kCFAllocatorDefault,
1180  kCFNumberIntType,
1181  &avpar->num);
1182 
1183  den = CFNumberCreate(kCFAllocatorDefault,
1184  kCFNumberIntType,
1185  &avpar->den);
1186 
1187 
1188 
1189  par = CFDictionaryCreateMutable(kCFAllocatorDefault,
1190  2,
1191  &kCFCopyStringDictionaryKeyCallBacks,
1192  &kCFTypeDictionaryValueCallBacks);
1193 
1194  if (!par || !num || !den) {
1195  if (par) CFRelease(par);
1196  if (num) CFRelease(num);
1197  if (den) CFRelease(den);
1198 
1199  return AVERROR(ENOMEM);
1200  }
1201 
1202  CFDictionarySetValue(
1203  par,
1204  kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing,
1205  num);
1206 
1207  CFDictionarySetValue(
1208  par,
1209  kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing,
1210  den);
1211 
1212  status = VTSessionSetProperty(vtctx->session,
1213  kVTCompressionPropertyKey_PixelAspectRatio,
1214  par);
1215 
1216  CFRelease(par);
1217  CFRelease(num);
1218  CFRelease(den);
1219 
1220  if (status) {
1221  av_log(avctx,
1222  AV_LOG_ERROR,
1223  "Error setting pixel aspect ratio to %d:%d: %d.\n",
1224  avctx->sample_aspect_ratio.num,
1225  avctx->sample_aspect_ratio.den,
1226  status);
1227 
1228  return AVERROR_EXTERNAL;
1229  }
1230  }
1231 
1232 
1233  if (vtctx->transfer_function) {
1234  status = VTSessionSetProperty(vtctx->session,
1235  kVTCompressionPropertyKey_TransferFunction,
1236  vtctx->transfer_function);
1237 
1238  if (status) {
1239  av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status);
1240  }
1241  }
1242 
1243 
1244  if (vtctx->ycbcr_matrix) {
1245  status = VTSessionSetProperty(vtctx->session,
1246  kVTCompressionPropertyKey_YCbCrMatrix,
1247  vtctx->ycbcr_matrix);
1248 
1249  if (status) {
1250  av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status);
1251  }
1252  }
1253 
1254 
1255  if (vtctx->color_primaries) {
1256  status = VTSessionSetProperty(vtctx->session,
1257  kVTCompressionPropertyKey_ColorPrimaries,
1258  vtctx->color_primaries);
1259 
1260  if (status) {
1261  av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status);
1262  }
1263  }
1264 
1265  if (gamma_level) {
1266  status = VTSessionSetProperty(vtctx->session,
1267  kCVImageBufferGammaLevelKey,
1268  gamma_level);
1269 
1270  if (status) {
1271  av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status);
1272  }
1273  }
1274 
1275  if (!vtctx->has_b_frames) {
1276  status = VTSessionSetProperty(vtctx->session,
1277  kVTCompressionPropertyKey_AllowFrameReordering,
1278  kCFBooleanFalse);
1279 
1280  if (status) {
1281  av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status);
1282  return AVERROR_EXTERNAL;
1283  }
1284  }
1285 
1286  if (vtctx->entropy != VT_ENTROPY_NOT_SET) {
1287  CFStringRef entropy = vtctx->entropy == VT_CABAC ?
1288  compat_keys.kVTH264EntropyMode_CABAC:
1289  compat_keys.kVTH264EntropyMode_CAVLC;
1290 
1291  status = VTSessionSetProperty(vtctx->session,
1292  compat_keys.kVTCompressionPropertyKey_H264EntropyMode,
1293  entropy);
1294 
1295  if (status) {
1296  av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status);
1297  }
1298  }
1299 
1300  if (vtctx->realtime) {
1301  status = VTSessionSetProperty(vtctx->session,
1302  compat_keys.kVTCompressionPropertyKey_RealTime,
1303  kCFBooleanTrue);
1304 
1305  if (status) {
1306  av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status);
1307  }
1308  }
1309 
1310  status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
1311  if (status) {
1312  av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status);
1313  return AVERROR_EXTERNAL;
1314  }
1315 
1316  return 0;
1317 }
1318 
1319 static int vtenc_configure_encoder(AVCodecContext *avctx)
1320 {
1321  CFMutableDictionaryRef enc_info;
1322  CFMutableDictionaryRef pixel_buffer_info;
1323  CMVideoCodecType codec_type;
1324  VTEncContext *vtctx = avctx->priv_data;
1325  CFStringRef profile_level;
1326  CFNumberRef gamma_level = NULL;
1327  int status;
1328 
1329  codec_type = get_cm_codec_type(avctx->codec_id);
1330  if (!codec_type) {
1331  av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
1332  return AVERROR(EINVAL);
1333  }
1334 
1335  vtctx->codec_id = avctx->codec_id;
1336 
1337  if (vtctx->codec_id == AV_CODEC_ID_H264) {
1338  vtctx->get_param_set_func = CMVideoFormatDescriptionGetH264ParameterSetAtIndex;
1339 
1340  vtctx->has_b_frames = avctx->max_b_frames > 0;
1341  if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){
1342  av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n");
1343  vtctx->has_b_frames = false;
1344  }
1345 
1346  if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) {
1347  av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n");
1348  vtctx->entropy = VT_ENTROPY_NOT_SET;
1349  }
1350 
1351  if (!get_vt_h264_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1352  } else {
1353  vtctx->get_param_set_func = compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
1354  if (!vtctx->get_param_set_func) return AVERROR(EINVAL);
1355  if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1356  }
1357 
1358  enc_info = CFDictionaryCreateMutable(
1359  kCFAllocatorDefault,
1360  20,
1361  &kCFCopyStringDictionaryKeyCallBacks,
1362  &kCFTypeDictionaryValueCallBacks
1363  );
1364 
1365  if (!enc_info) return AVERROR(ENOMEM);
1366 
1367 #if !TARGET_OS_IPHONE
1368  if(vtctx->require_sw) {
1369  CFDictionarySetValue(enc_info,
1370  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1371  kCFBooleanFalse);
1372  } else if (!vtctx->allow_sw) {
1373  CFDictionarySetValue(enc_info,
1374  compat_keys.kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
1375  kCFBooleanTrue);
1376  } else {
1377  CFDictionarySetValue(enc_info,
1378  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1379  kCFBooleanTrue);
1380  }
1381 #endif
1382 
1383  if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) {
1384  status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info);
1385  if (status)
1386  goto init_cleanup;
1387  } else {
1388  pixel_buffer_info = NULL;
1389  }
1390 
1391  vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
1392 
1393  get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
1394  get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix);
1395  get_cv_color_primaries(avctx, &vtctx->color_primaries);
1396 
1397 
1398  if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
1399  status = vtenc_populate_extradata(avctx,
1400  codec_type,
1401  profile_level,
1402  gamma_level,
1403  enc_info,
1404  pixel_buffer_info);
1405  if (status)
1406  goto init_cleanup;
1407  }
1408 
1409  status = vtenc_create_encoder(avctx,
1410  codec_type,
1411  profile_level,
1412  gamma_level,
1413  enc_info,
1414  pixel_buffer_info,
1415  &vtctx->session);
1416 
1417 init_cleanup:
1418  if (gamma_level)
1419  CFRelease(gamma_level);
1420 
1421  if (pixel_buffer_info)
1422  CFRelease(pixel_buffer_info);
1423 
1424  CFRelease(enc_info);
1425 
1426  return status;
1427 }
1428 
1429 static av_cold int vtenc_init(AVCodecContext *avctx)
1430 {
1431  VTEncContext *vtctx = avctx->priv_data;
1432  CFBooleanRef has_b_frames_cfbool;
1433  int status;
1434 
1435  pthread_once(&once_ctrl, loadVTEncSymbols);
1436 
1437  pthread_mutex_init(&vtctx->lock, NULL);
1438  pthread_cond_init(&vtctx->cv_sample_sent, NULL);
1439 
1440  vtctx->session = NULL;
1441  status = vtenc_configure_encoder(avctx);
1442  if (status) return status;
1443 
1444  status = VTSessionCopyProperty(vtctx->session,
1445  kVTCompressionPropertyKey_AllowFrameReordering,
1446  kCFAllocatorDefault,
1447  &has_b_frames_cfbool);
1448 
1449  if (!status && has_b_frames_cfbool) {
1450  //Some devices don't output B-frames for main profile, even if requested.
1451  vtctx->has_b_frames = CFBooleanGetValue(has_b_frames_cfbool);
1452  CFRelease(has_b_frames_cfbool);
1453  }
1454  avctx->has_b_frames = vtctx->has_b_frames;
1455 
1456  return 0;
1457 }
1458 
1459 static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
1460 {
1461  CFArrayRef attachments;
1462  CFDictionaryRef attachment;
1463  CFBooleanRef not_sync;
1464  CFIndex len;
1465 
1466  attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false);
1467  len = !attachments ? 0 : CFArrayGetCount(attachments);
1468 
1469  if (!len) {
1470  *is_key_frame = true;
1471  return;
1472  }
1473 
1474  attachment = CFArrayGetValueAtIndex(attachments, 0);
1475 
1476  if (CFDictionaryGetValueIfPresent(attachment,
1477  kCMSampleAttachmentKey_NotSync,
1478  (const void **)&not_sync))
1479  {
1480  *is_key_frame = !CFBooleanGetValue(not_sync);
1481  } else {
1482  *is_key_frame = true;
1483  }
1484 }
1485 
1486 static int is_post_sei_nal_type(int nal_type){
1487  return nal_type != H264_NAL_SEI &&
1488  nal_type != H264_NAL_SPS &&
1489  nal_type != H264_NAL_PPS &&
1490  nal_type != H264_NAL_AUD;
1491 }
1492 
1493 /*
1494  * Finds the sei message start/size of type find_sei_type.
1495  * If more than one of that type exists, the last one is returned.
1496  */
1497 static int find_sei_end(AVCodecContext *avctx,
1498  uint8_t *nal_data,
1499  size_t nal_size,
1500  uint8_t **sei_end)
1501 {
1502  int nal_type;
1503  size_t sei_payload_size = 0;
1504  int sei_payload_type = 0;
1505  *sei_end = NULL;
1506  uint8_t *nal_start = nal_data;
1507 
1508  if (!nal_size)
1509  return 0;
1510 
1511  nal_type = *nal_data & 0x1F;
1512  if (nal_type != H264_NAL_SEI)
1513  return 0;
1514 
1515  nal_data++;
1516  nal_size--;
1517 
1518  if (nal_data[nal_size - 1] == 0x80)
1519  nal_size--;
1520 
1521  while (nal_size > 0 && *nal_data > 0) {
1522  do{
1523  sei_payload_type += *nal_data;
1524  nal_data++;
1525  nal_size--;
1526  } while (nal_size > 0 && *nal_data == 0xFF);
1527 
1528  if (!nal_size) {
1529  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing type.\n");
1530  return AVERROR_INVALIDDATA;
1531  }
1532 
1533  do{
1534  sei_payload_size += *nal_data;
1535  nal_data++;
1536  nal_size--;
1537  } while (nal_size > 0 && *nal_data == 0xFF);
1538 
1539  if (nal_size < sei_payload_size) {
1540  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing size.\n");
1541  return AVERROR_INVALIDDATA;
1542  }
1543 
1544  nal_data += sei_payload_size;
1545  nal_size -= sei_payload_size;
1546  }
1547 
1548  *sei_end = nal_data;
1549 
1550  return nal_data - nal_start + 1;
1551 }
1552 
1553 /**
1554  * Copies the data inserting emulation prevention bytes as needed.
1555  * Existing data in the destination can be taken into account by providing
1556  * dst with a dst_offset > 0.
1557  *
1558  * @return The number of bytes copied on success. On failure, the negative of
1559  * the number of bytes needed to copy src is returned.
1560  */
1561 static int copy_emulation_prev(const uint8_t *src,
1562  size_t src_size,
1563  uint8_t *dst,
1564  ssize_t dst_offset,
1565  size_t dst_size)
1566 {
1567  int zeros = 0;
1568  int wrote_bytes;
1569  uint8_t* dst_start;
1570  uint8_t* dst_end = dst + dst_size;
1571  const uint8_t* src_end = src + src_size;
1572  int start_at = dst_offset > 2 ? dst_offset - 2 : 0;
1573  int i;
1574  for (i = start_at; i < dst_offset && i < dst_size; i++) {
1575  if (!dst[i])
1576  zeros++;
1577  else
1578  zeros = 0;
1579  }
1580 
1581  dst += dst_offset;
1582  dst_start = dst;
1583  for (; src < src_end; src++, dst++) {
1584  if (zeros == 2) {
1585  int insert_ep3_byte = *src <= 3;
1586  if (insert_ep3_byte) {
1587  if (dst < dst_end)
1588  *dst = 3;
1589  dst++;
1590  }
1591 
1592  zeros = 0;
1593  }
1594 
1595  if (dst < dst_end)
1596  *dst = *src;
1597 
1598  if (!*src)
1599  zeros++;
1600  else
1601  zeros = 0;
1602  }
1603 
1604  wrote_bytes = dst - dst_start;
1605 
1606  if (dst > dst_end)
1607  return -wrote_bytes;
1608 
1609  return wrote_bytes;
1610 }
1611 
1612 static int write_sei(const ExtraSEI *sei,
1613  int sei_type,
1614  uint8_t *dst,
1615  size_t dst_size)
1616 {
1617  uint8_t *sei_start = dst;
1618  size_t remaining_sei_size = sei->size;
1619  size_t remaining_dst_size = dst_size;
1620  int header_bytes;
1621  int bytes_written;
1622  ssize_t offset;
1623 
1624  if (!remaining_dst_size)
1625  return AVERROR_BUFFER_TOO_SMALL;
1626 
1627  while (sei_type && remaining_dst_size != 0) {
1628  int sei_byte = sei_type > 255 ? 255 : sei_type;
1629  *dst = sei_byte;
1630 
1631  sei_type -= sei_byte;
1632  dst++;
1633  remaining_dst_size--;
1634  }
1635 
1636  if (!dst_size)
1637  return AVERROR_BUFFER_TOO_SMALL;
1638 
1639  while (remaining_sei_size && remaining_dst_size != 0) {
1640  int size_byte = remaining_sei_size > 255 ? 255 : remaining_sei_size;
1641  *dst = size_byte;
1642 
1643  remaining_sei_size -= size_byte;
1644  dst++;
1645  remaining_dst_size--;
1646  }
1647 
1648  if (remaining_dst_size < sei->size)
1649  return AVERROR_BUFFER_TOO_SMALL;
1650 
1651  header_bytes = dst - sei_start;
1652 
1653  offset = header_bytes;
1654  bytes_written = copy_emulation_prev(sei->data,
1655  sei->size,
1656  sei_start,
1657  offset,
1658  dst_size);
1659  if (bytes_written < 0)
1660  return AVERROR_BUFFER_TOO_SMALL;
1661 
1662  bytes_written += header_bytes;
1663  return bytes_written;
1664 }
1665 
1666 /**
1667  * Copies NAL units and replaces length codes with
1668  * H.264 Annex B start codes. On failure, the contents of
1669  * dst_data may have been modified.
1670  *
1671  * @param length_code_size Byte length of each length code
1672  * @param sample_buffer NAL units prefixed with length codes.
1673  * @param sei Optional A53 closed captions SEI data.
1674  * @param dst_data Must be zeroed before calling this function.
1675  * Contains the copied NAL units prefixed with
1676  * start codes when the function returns
1677  * successfully.
1678  * @param dst_size Length of dst_data
1679  * @return 0 on success
1680  * AVERROR_INVALIDDATA if length_code_size is invalid
1681  * AVERROR_BUFFER_TOO_SMALL if dst_data is too small
1682  * or if a length_code in src_data specifies data beyond
1683  * the end of its buffer.
1684  */
1685 static int copy_replace_length_codes(
1686  AVCodecContext *avctx,
1687  size_t length_code_size,
1688  CMSampleBufferRef sample_buffer,
1689  ExtraSEI *sei,
1690  uint8_t *dst_data,
1691  size_t dst_size)
1692 {
1693  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1694  size_t remaining_src_size = src_size;
1695  size_t remaining_dst_size = dst_size;
1696  size_t src_offset = 0;
1697  int wrote_sei = 0;
1698  int status;
1699  uint8_t size_buf[4];
1700  uint8_t nal_type;
1701  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
1702 
1703  if (length_code_size > 4) {
1704  return AVERROR_INVALIDDATA;
1705  }
1706 
1707  while (remaining_src_size > 0) {
1708  size_t curr_src_len;
1709  size_t curr_dst_len;
1710  size_t box_len = 0;
1711  size_t i;
1712 
1713  uint8_t *dst_box;
1714 
1715  status = CMBlockBufferCopyDataBytes(block,
1716  src_offset,
1717  length_code_size,
1718  size_buf);
1719  if (status) {
1720  av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status);
1721  return AVERROR_EXTERNAL;
1722  }
1723 
1724  status = CMBlockBufferCopyDataBytes(block,
1725  src_offset + length_code_size,
1726  1,
1727  &nal_type);
1728 
1729  if (status) {
1730  av_log(avctx, AV_LOG_ERROR, "Cannot copy type: %d\n", status);
1731  return AVERROR_EXTERNAL;
1732  }
1733 
1734  nal_type &= 0x1F;
1735 
1736  for (i = 0; i < length_code_size; i++) {
1737  box_len <<= 8;
1738  box_len |= size_buf[i];
1739  }
1740 
1741  if (sei && !wrote_sei && is_post_sei_nal_type(nal_type)) {
1742  //No SEI NAL unit - insert.
1743  int wrote_bytes;
1744 
1745  memcpy(dst_data, start_code, sizeof(start_code));
1746  dst_data += sizeof(start_code);
1747  remaining_dst_size -= sizeof(start_code);
1748 
1749  *dst_data = H264_NAL_SEI;
1750  dst_data++;
1751  remaining_dst_size--;
1752 
1753  wrote_bytes = write_sei(sei,
1754  H264_SEI_TYPE_USER_DATA_REGISTERED,
1755  dst_data,
1756  remaining_dst_size);
1757 
1758  if (wrote_bytes < 0)
1759  return wrote_bytes;
1760 
1761  remaining_dst_size -= wrote_bytes;
1762  dst_data += wrote_bytes;
1763 
1764  if (remaining_dst_size <= 0)
1765  return AVERROR_BUFFER_TOO_SMALL;
1766 
1767  *dst_data = 0x80;
1768 
1769  dst_data++;
1770  remaining_dst_size--;
1771 
1772  wrote_sei = 1;
1773  }
1774 
1775  curr_src_len = box_len + length_code_size;
1776  curr_dst_len = box_len + sizeof(start_code);
1777 
1778  if (remaining_src_size < curr_src_len) {
1779  return AVERROR_BUFFER_TOO_SMALL;
1780  }
1781 
1782  if (remaining_dst_size < curr_dst_len) {
1783  return AVERROR_BUFFER_TOO_SMALL;
1784  }
1785 
1786  dst_box = dst_data + sizeof(start_code);
1787 
1788  memcpy(dst_data, start_code, sizeof(start_code));
1789  status = CMBlockBufferCopyDataBytes(block,
1790  src_offset + length_code_size,
1791  box_len,
1792  dst_box);
1793 
1794  if (status) {
1795  av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status);
1796  return AVERROR_EXTERNAL;
1797  }
1798 
1799  if (sei && !wrote_sei && nal_type == H264_NAL_SEI) {
1800  //Found SEI NAL unit - append.
1801  int wrote_bytes;
1802  int old_sei_length;
1803  int extra_bytes;
1804  uint8_t *new_sei;
1805  old_sei_length = find_sei_end(avctx, dst_box, box_len, &new_sei);
1806  if (old_sei_length < 0)
1807  return status;
1808 
1809  wrote_bytes = write_sei(sei,
1810  H264_SEI_TYPE_USER_DATA_REGISTERED,
1811  new_sei,
1812  remaining_dst_size - old_sei_length);
1813  if (wrote_bytes < 0)
1814  return wrote_bytes;
1815 
1816  if (new_sei + wrote_bytes >= dst_data + remaining_dst_size)
1817  return AVERROR_BUFFER_TOO_SMALL;
1818 
1819  new_sei[wrote_bytes++] = 0x80;
1820  extra_bytes = wrote_bytes - (dst_box + box_len - new_sei);
1821 
1822  dst_data += extra_bytes;
1823  remaining_dst_size -= extra_bytes;
1824 
1825  wrote_sei = 1;
1826  }
1827 
1828  src_offset += curr_src_len;
1829  dst_data += curr_dst_len;
1830 
1831  remaining_src_size -= curr_src_len;
1832  remaining_dst_size -= curr_dst_len;
1833  }
1834 
1835  return 0;
1836 }
1837 
1838 /**
1839  * Returns a sufficient number of bytes to contain the sei data.
1840  * It may be greater than the minimum required.
1841  */
1842 static int get_sei_msg_bytes(const ExtraSEI* sei, int type){
1843  int copied_size;
1844  if (sei->size == 0)
1845  return 0;
1846 
1847  copied_size = -copy_emulation_prev(sei->data,
1848  sei->size,
1849  NULL,
1850  0,
1851  0);
1852 
1853  if ((sei->size % 255) == 0) //may result in an extra byte
1854  copied_size++;
1855 
1856  return copied_size + sei->size / 255 + 1 + type / 255 + 1;
1857 }
1858 
1859 static int vtenc_cm_to_avpacket(
1860  AVCodecContext *avctx,
1861  CMSampleBufferRef sample_buffer,
1862  AVPacket *pkt,
1863  ExtraSEI *sei)
1864 {
1865  VTEncContext *vtctx = avctx->priv_data;
1866 
1867  int status;
1868  bool is_key_frame;
1869  bool add_header;
1870  size_t length_code_size;
1871  size_t header_size = 0;
1872  size_t in_buf_size;
1873  size_t out_buf_size;
1874  size_t sei_nalu_size = 0;
1875  int64_t dts_delta;
1876  int64_t time_base_num;
1877  int nalu_count;
1878  CMTime pts;
1879  CMTime dts;
1880  CMVideoFormatDescriptionRef vid_fmt;
1881 
1882 
1883  vtenc_get_frame_info(sample_buffer, &is_key_frame);
1884  status = get_length_code_size(avctx, sample_buffer, &length_code_size);
1885  if (status) return status;
1886 
1887  add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
1888 
1889  if (add_header) {
1890  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
1891  if (!vid_fmt) {
1892  av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
1893  return AVERROR_EXTERNAL;
1894  }
1895 
1896  int status = get_params_size(avctx, vid_fmt, &header_size);
1897  if (status) return status;
1898  }
1899 
1900  status = count_nalus(length_code_size, sample_buffer, &nalu_count);
1901  if(status)
1902  return status;
1903 
1904  if (sei) {
1905  size_t msg_size = get_sei_msg_bytes(sei,
1906  H264_SEI_TYPE_USER_DATA_REGISTERED);
1907 
1908  sei_nalu_size = sizeof(start_code) + 1 + msg_size + 1;
1909  }
1910 
1911  in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1912  out_buf_size = header_size +
1913  in_buf_size +
1914  sei_nalu_size +
1915  nalu_count * ((int)sizeof(start_code) - (int)length_code_size);
1916 
1917  status = ff_alloc_packet2(avctx, pkt, out_buf_size, out_buf_size);
1918  if (status < 0)
1919  return status;
1920 
1921  if (add_header) {
1922  status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size);
1923  if(status) return status;
1924  }
1925 
1926  status = copy_replace_length_codes(
1927  avctx,
1928  length_code_size,
1929  sample_buffer,
1930  sei,
1931  pkt->data + header_size,
1932  pkt->size - header_size
1933  );
1934 
1935  if (status) {
1936  av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status);
1937  return status;
1938  }
1939 
1940  if (is_key_frame) {
1941  pkt->flags |= AV_PKT_FLAG_KEY;
1942  }
1943 
1944  pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
1945  dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer);
1946 
1947  if (CMTIME_IS_INVALID(dts)) {
1948  if (!vtctx->has_b_frames) {
1949  dts = pts;
1950  } else {
1951  av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n");
1952  return AVERROR_EXTERNAL;
1953  }
1954  }
1955 
1956  dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
1957  time_base_num = avctx->time_base.num;
1958  pkt->pts = pts.value / time_base_num;
1959  pkt->dts = dts.value / time_base_num - dts_delta;
1960  pkt->size = out_buf_size;
1961 
1962  return 0;
1963 }
1964 
1965 /*
1966  * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
1967  * containing all planes if so.
1968  */
1969 static int get_cv_pixel_info(
1970  AVCodecContext *avctx,
1971  const AVFrame *frame,
1972  int *color,
1973  int *plane_count,
1974  size_t *widths,
1975  size_t *heights,
1976  size_t *strides,
1977  size_t *contiguous_buf_size)
1978 {
1979  VTEncContext *vtctx = avctx->priv_data;
1980  int av_format = frame->format;
1981  int av_color_range = frame->color_range;
1982  int i;
1983  int range_guessed;
1984  int status;
1985 
1986  status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
1987  if (status) {
1988  av_log(avctx,
1989  AV_LOG_ERROR,
1990  "Could not get pixel format for color format '%s' range '%s'.\n",
1991  av_get_pix_fmt_name(av_format),
1992  av_color_range > AVCOL_RANGE_UNSPECIFIED &&
1993  av_color_range < AVCOL_RANGE_NB ?
1994  av_color_range_name(av_color_range) :
1995  "Unknown");
1996 
1997  return AVERROR(EINVAL);
1998  }
1999 
2000  if (range_guessed) {
2001  if (!vtctx->warned_color_range) {
2002  vtctx->warned_color_range = true;
2003  av_log(avctx,
2004  AV_LOG_WARNING,
2005  "Color range not set for %s. Using MPEG range.\n",
2006  av_get_pix_fmt_name(av_format));
2007  }
2008  }
2009 
2010  switch (av_format) {
2011  case AV_PIX_FMT_NV12:
2012  *plane_count = 2;
2013 
2014  widths [0] = avctx->width;
2015  heights[0] = avctx->height;
2016  strides[0] = frame ? frame->linesize[0] : avctx->width;
2017 
2018  widths [1] = (avctx->width + 1) / 2;
2019  heights[1] = (avctx->height + 1) / 2;
2020  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) & -2;
2021  break;
2022 
2023  case AV_PIX_FMT_YUV420P:
2024  *plane_count = 3;
2025 
2026  widths [0] = avctx->width;
2027  heights[0] = avctx->height;
2028  strides[0] = frame ? frame->linesize[0] : avctx->width;
2029 
2030  widths [1] = (avctx->width + 1) / 2;
2031  heights[1] = (avctx->height + 1) / 2;
2032  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
2033 
2034  widths [2] = (avctx->width + 1) / 2;
2035  heights[2] = (avctx->height + 1) / 2;
2036  strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
2037  break;
2038 
2039  case AV_PIX_FMT_P010LE:
2040  *plane_count = 2;
2041  widths[0] = avctx->width;
2042  heights[0] = avctx->height;
2043  strides[0] = frame ? frame->linesize[0] : (avctx->width * 2 + 63) & -64;
2044 
2045  widths[1] = (avctx->width + 1) / 2;
2046  heights[1] = (avctx->height + 1) / 2;
2047  strides[1] = frame ? frame->linesize[1] : ((avctx->width + 1) / 2 + 63) & -64;
2048  break;
2049 
2050  default:
2051  av_log(
2052  avctx,
2053  AV_LOG_ERROR,
2054  "Could not get frame format info for color %d range %d.\n",
2055  av_format,
2056  av_color_range);
2057 
2058  return AVERROR(EINVAL);
2059  }
2060 
2061  *contiguous_buf_size = 0;
2062  for (i = 0; i < *plane_count; i++) {
2063  if (i < *plane_count - 1 &&
2064  frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
2065  *contiguous_buf_size = 0;
2066  break;
2067  }
2068 
2069  *contiguous_buf_size += strides[i] * heights[i];
2070  }
2071 
2072  return 0;
2073 }
2074 
2075 //Not used on OSX - frame is never copied.
2076 static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx,
2077  const AVFrame *frame,
2078  CVPixelBufferRef cv_img,
2079  const size_t *plane_strides,
2080  const size_t *plane_rows)
2081 {
2082  int i, j;
2083  size_t plane_count;
2084  int status;
2085  int rows;
2086  int src_stride;
2087  int dst_stride;
2088  uint8_t *src_addr;
2089  uint8_t *dst_addr;
2090  size_t copy_bytes;
2091 
2092  status = CVPixelBufferLockBaseAddress(cv_img, 0);
2093  if (status) {
2094  av_log(
2095  avctx,
2096  AV_LOG_ERROR,
2097  "Error: Could not lock base address of CVPixelBuffer: %d.\n",
2098  status
2099  );
2100  }
2101 
2102  if (CVPixelBufferIsPlanar(cv_img)) {
2103  plane_count = CVPixelBufferGetPlaneCount(cv_img);
2104  for (i = 0; frame->data[i]; i++) {
2105  if (i == plane_count) {
2106  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2107  av_log(avctx,
2108  AV_LOG_ERROR,
2109  "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
2110  );
2111 
2112  return AVERROR_EXTERNAL;
2113  }
2114 
2115  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
2116  src_addr = (uint8_t*)frame->data[i];
2117  dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
2118  src_stride = plane_strides[i];
2119  rows = plane_rows[i];
2120 
2121  if (dst_stride == src_stride) {
2122  memcpy(dst_addr, src_addr, src_stride * rows);
2123  } else {
2124  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2125 
2126  for (j = 0; j < rows; j++) {
2127  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2128  }
2129  }
2130  }
2131  } else {
2132  if (frame->data[1]) {
2133  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2134  av_log(avctx,
2135  AV_LOG_ERROR,
2136  "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
2137  );
2138 
2139  return AVERROR_EXTERNAL;
2140  }
2141 
2142  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
2143  src_addr = (uint8_t*)frame->data[0];
2144  dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
2145  src_stride = plane_strides[0];
2146  rows = plane_rows[0];
2147 
2148  if (dst_stride == src_stride) {
2149  memcpy(dst_addr, src_addr, src_stride * rows);
2150  } else {
2151  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2152 
2153  for (j = 0; j < rows; j++) {
2154  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2155  }
2156  }
2157  }
2158 
2159  status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
2160  if (status) {
2161  av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status);
2162  return AVERROR_EXTERNAL;
2163  }
2164 
2165  return 0;
2166 }
2167 
2168 static int create_cv_pixel_buffer(AVCodecContext *avctx,
2169  const AVFrame *frame,
2170  CVPixelBufferRef *cv_img)
2171 {
2172  int plane_count;
2173  int color;
2174  size_t widths [AV_NUM_DATA_POINTERS];
2175  size_t heights[AV_NUM_DATA_POINTERS];
2176  size_t strides[AV_NUM_DATA_POINTERS];
2177  int status;
2178  size_t contiguous_buf_size;
2179  CVPixelBufferPoolRef pix_buf_pool;
2180  VTEncContext* vtctx = avctx->priv_data;
2181 
2182  if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) {
2183  av_assert0(frame->format == AV_PIX_FMT_VIDEOTOOLBOX);
2184 
2185  *cv_img = (CVPixelBufferRef)frame->data[3];
2186  av_assert0(*cv_img);
2187 
2188  CFRetain(*cv_img);
2189  return 0;
2190  }
2191 
2192  memset(widths, 0, sizeof(widths));
2193  memset(heights, 0, sizeof(heights));
2194  memset(strides, 0, sizeof(strides));
2195 
2196  status = get_cv_pixel_info(
2197  avctx,
2198  frame,
2199  &color,
2200  &plane_count,
2201  widths,
2202  heights,
2203  strides,
2204  &contiguous_buf_size
2205  );
2206 
2207  if (status) {
2208  av_log(
2209  avctx,
2210  AV_LOG_ERROR,
2211  "Error: Cannot convert format %d color_range %d: %d\n",
2212  frame->format,
2213  frame->color_range,
2214  status
2215  );
2216 
2217  return AVERROR_EXTERNAL;
2218  }
2219 
2220  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2221  if (!pix_buf_pool) {
2222  /* On iOS, the VT session is invalidated when the APP switches from
2223  * foreground to background and vice versa. Fetch the actual error code
2224  * of the VT session to detect that case and restart the VT session
2225  * accordingly. */
2226  OSStatus vtstatus;
2227 
2228  vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
2229  if (vtstatus == kVTInvalidSessionErr) {
2230  CFRelease(vtctx->session);
2231  vtctx->session = NULL;
2232  status = vtenc_configure_encoder(avctx);
2233  if (status == 0)
2234  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2235  }
2236  if (!pix_buf_pool) {
2237  av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
2238  return AVERROR_EXTERNAL;
2239  }
2240  else
2241  av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
2242  "kVTInvalidSessionErr error.\n");
2243  }
2244 
2245  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2246  pix_buf_pool,
2247  cv_img);
2248 
2249 
2250  if (status) {
2251  av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status);
2252  return AVERROR_EXTERNAL;
2253  }
2254 
2255  status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
2256  if (status) {
2257  CFRelease(*cv_img);
2258  *cv_img = NULL;
2259  return status;
2260  }
2261 
2262  return 0;
2263 }
2264 
2265 static int create_encoder_dict_h264(const AVFrame *frame,
2266  CFDictionaryRef* dict_out)
2267 {
2268  CFDictionaryRef dict = NULL;
2269  if (frame->pict_type == AV_PICTURE_TYPE_I) {
2270  const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
2271  const void *vals[] = { kCFBooleanTrue };
2272 
2273  dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL);
2274  if(!dict) return AVERROR(ENOMEM);
2275  }
2276 
2277  *dict_out = dict;
2278  return 0;
2279 }
2280 
2281 static int vtenc_send_frame(AVCodecContext *avctx,
2282  VTEncContext *vtctx,
2283  const AVFrame *frame)
2284 {
2285  CMTime time;
2286  CFDictionaryRef frame_dict;
2287  CVPixelBufferRef cv_img = NULL;
2288  AVFrameSideData *side_data = NULL;
2289  ExtraSEI *sei = NULL;
2290  int status = create_cv_pixel_buffer(avctx, frame, &cv_img);
2291 
2292  if (status) return status;
2293 
2294  status = create_encoder_dict_h264(frame, &frame_dict);
2295  if (status) {
2296  CFRelease(cv_img);
2297  return status;
2298  }
2299 
2300  side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
2301  if (vtctx->a53_cc && side_data && side_data->size) {
2302  sei = av_mallocz(sizeof(*sei));
2303  if (!sei) {
2304  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2305  } else {
2306  int ret = ff_alloc_a53_sei(frame, 0, &sei->data, &sei->size);
2307  if (ret < 0) {
2308  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2309  av_free(sei);
2310  sei = NULL;
2311  }
2312  }
2313  }
2314 
2315  time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den);
2316  status = VTCompressionSessionEncodeFrame(
2317  vtctx->session,
2318  cv_img,
2319  time,
2320  kCMTimeInvalid,
2321  frame_dict,
2322  sei,
2323  NULL
2324  );
2325 
2326  if (frame_dict) CFRelease(frame_dict);
2327  CFRelease(cv_img);
2328 
2329  if (status) {
2330  av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status);
2331  return AVERROR_EXTERNAL;
2332  }
2333 
2334  return 0;
2335 }
2336 
2337 static av_cold int vtenc_frame(
2338  AVCodecContext *avctx,
2339  AVPacket *pkt,
2340  const AVFrame *frame,
2341  int *got_packet)
2342 {
2343  VTEncContext *vtctx = avctx->priv_data;
2344  bool get_frame;
2345  int status;
2346  CMSampleBufferRef buf = NULL;
2347  ExtraSEI *sei = NULL;
2348 
2349  if (frame) {
2350  status = vtenc_send_frame(avctx, vtctx, frame);
2351 
2352  if (status) {
2353  status = AVERROR_EXTERNAL;
2354  goto end_nopkt;
2355  }
2356 
2357  if (vtctx->frame_ct_in == 0) {
2358  vtctx->first_pts = frame->pts;
2359  } else if(vtctx->frame_ct_in == 1 && vtctx->has_b_frames) {
2360  vtctx->dts_delta = frame->pts - vtctx->first_pts;
2361  }
2362 
2363  vtctx->frame_ct_in++;
2364  } else if(!vtctx->flushing) {
2365  vtctx->flushing = true;
2366 
2367  status = VTCompressionSessionCompleteFrames(vtctx->session,
2368  kCMTimeIndefinite);
2369 
2370  if (status) {
2371  av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status);
2372  status = AVERROR_EXTERNAL;
2373  goto end_nopkt;
2374  }
2375  }
2376 
2377  *got_packet = 0;
2378  get_frame = vtctx->dts_delta >= 0 || !frame;
2379  if (!get_frame) {
2380  status = 0;
2381  goto end_nopkt;
2382  }
2383 
2384  status = vtenc_q_pop(vtctx, !frame, &buf, &sei);
2385  if (status) goto end_nopkt;
2386  if (!buf) goto end_nopkt;
2387 
2388  status = vtenc_cm_to_avpacket(avctx, buf, pkt, sei);
2389  if (sei) {
2390  if (sei->data) av_free(sei->data);
2391  av_free(sei);
2392  }
2393  CFRelease(buf);
2394  if (status) goto end_nopkt;
2395 
2396  *got_packet = 1;
2397  return 0;
2398 
2399 end_nopkt:
2400  av_packet_unref(pkt);
2401  return status;
2402 }
2403 
2404 static int vtenc_populate_extradata(AVCodecContext *avctx,
2405  CMVideoCodecType codec_type,
2406  CFStringRef profile_level,
2407  CFNumberRef gamma_level,
2408  CFDictionaryRef enc_info,
2409  CFDictionaryRef pixel_buffer_info)
2410 {
2411  VTEncContext *vtctx = avctx->priv_data;
2412  int status;
2413  CVPixelBufferPoolRef pool = NULL;
2414  CVPixelBufferRef pix_buf = NULL;
2415  CMTime time;
2416  CMSampleBufferRef buf = NULL;
2417 
2418  status = vtenc_create_encoder(avctx,
2419  codec_type,
2420  profile_level,
2421  gamma_level,
2422  enc_info,
2423  pixel_buffer_info,
2424  &vtctx->session);
2425  if (status)
2426  goto pe_cleanup;
2427 
2428  pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2429  if(!pool){
2430  av_log(avctx, AV_LOG_ERROR, "Error getting pixel buffer pool.\n");
2431  goto pe_cleanup;
2432  }
2433 
2434  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2435  pool,
2436  &pix_buf);
2437 
2438  if(status != kCVReturnSuccess){
2439  av_log(avctx, AV_LOG_ERROR, "Error creating frame from pool: %d\n", status);
2440  goto pe_cleanup;
2441  }
2442 
2443  time = CMTimeMake(0, avctx->time_base.den);
2444  status = VTCompressionSessionEncodeFrame(vtctx->session,
2445  pix_buf,
2446  time,
2447  kCMTimeInvalid,
2448  NULL,
2449  NULL,
2450  NULL);
2451 
2452  if (status) {
2453  av_log(avctx,
2454  AV_LOG_ERROR,
2455  "Error sending frame for extradata: %d\n",
2456  status);
2457 
2458  goto pe_cleanup;
2459  }
2460 
2461  //Populates extradata - output frames are flushed and param sets are available.
2462  status = VTCompressionSessionCompleteFrames(vtctx->session,
2463  kCMTimeIndefinite);
2464 
2465  if (status)
2466  goto pe_cleanup;
2467 
2468  status = vtenc_q_pop(vtctx, 0, &buf, NULL);
2469  if (status) {
2470  av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status);
2471  goto pe_cleanup;
2472  }
2473 
2474  CFRelease(buf);
2475 
2476 
2477 
2478 pe_cleanup:
2479  if(vtctx->session)
2480  CFRelease(vtctx->session);
2481 
2482  vtctx->session = NULL;
2483  vtctx->frame_ct_out = 0;
2484 
2485  av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));
2486 
2487  return status;
2488 }
2489 
2490 static av_cold int vtenc_close(AVCodecContext *avctx)
2491 {
2492  VTEncContext *vtctx = avctx->priv_data;
2493 
2494  if(!vtctx->session) {
2495  pthread_cond_destroy(&vtctx->cv_sample_sent);
2496  pthread_mutex_destroy(&vtctx->lock);
2497  return 0;
2498  }
2499 
2500  VTCompressionSessionCompleteFrames(vtctx->session,
2501  kCMTimeIndefinite);
2502  clear_frame_queue(vtctx);
2503  pthread_cond_destroy(&vtctx->cv_sample_sent);
2504  pthread_mutex_destroy(&vtctx->lock);
2505  CFRelease(vtctx->session);
2506  vtctx->session = NULL;
2507 
2508  if (vtctx->color_primaries) {
2509  CFRelease(vtctx->color_primaries);
2510  vtctx->color_primaries = NULL;
2511  }
2512 
2513  if (vtctx->transfer_function) {
2514  CFRelease(vtctx->transfer_function);
2515  vtctx->transfer_function = NULL;
2516  }
2517 
2518  if (vtctx->ycbcr_matrix) {
2519  CFRelease(vtctx->ycbcr_matrix);
2520  vtctx->ycbcr_matrix = NULL;
2521  }
2522 
2523  return 0;
2524 }
2525 
2526 static const enum AVPixelFormat avc_pix_fmts[] = {
2527  AV_PIX_FMT_VIDEOTOOLBOX,
2528  AV_PIX_FMT_NV12,
2529  AV_PIX_FMT_YUV420P,
2530  AV_PIX_FMT_NONE
2531 };
2532 
2533 static const enum AVPixelFormat hevc_pix_fmts[] = {
2534  AV_PIX_FMT_VIDEOTOOLBOX,
2535  AV_PIX_FMT_NV12,
2536  AV_PIX_FMT_YUV420P,
2537  AV_PIX_FMT_P010LE,
2538  AV_PIX_FMT_NONE
2539 };
2540 
2541 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
2542 #define COMMON_OPTIONS \
2543  { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL, \
2544  { .i64 = 0 }, 0, 1, VE }, \
2545  { "require_sw", "Require software encoding", OFFSET(require_sw), AV_OPT_TYPE_BOOL, \
2546  { .i64 = 0 }, 0, 1, VE }, \
2547  { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).", \
2548  OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2549  { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.", \
2550  OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2551  { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.", \
2552  OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
2553 
2554 #define OFFSET(x) offsetof(VTEncContext, x)
2555 static const AVOption h264_options[] = {
2556  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" },
2557  { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" },
2558  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2559  { "high", "High Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH }, INT_MIN, INT_MAX, VE, "profile" },
2560  { "extended", "Extend Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_EXTENDED }, INT_MIN, INT_MAX, VE, "profile" },
2561 
2562  { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" },
2563  { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" },
2564  { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" },
2565  { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" },
2566  { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" },
2567  { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" },
2568  { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" },
2569  { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" },
2570  { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" },
2571  { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" },
2572  { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" },
2573 
2574  { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" },
2575  { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2576  { "vlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2577  { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2578  { "ac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2579 
2580  { "a53cc", "Use A53 Closed Captions (if available)", OFFSET(a53_cc), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, VE },
2581 
2582  COMMON_OPTIONS
2583  { NULL },
2584 };
2585 
2586 static const AVClass h264_videotoolbox_class = {
2587  .class_name = "h264_videotoolbox",
2588  .item_name = av_default_item_name,
2589  .option = h264_options,
2590  .version = LIBAVUTIL_VERSION_INT,
2591 };
2592 
2593 AVCodec ff_h264_videotoolbox_encoder = {
2594  .name = "h264_videotoolbox",
2595  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
2596  .type = AVMEDIA_TYPE_VIDEO,
2597  .id = AV_CODEC_ID_H264,
2598  .priv_data_size = sizeof(VTEncContext),
2599  .pix_fmts = avc_pix_fmts,
2600  .init = vtenc_init,
2601  .encode2 = vtenc_frame,
2602  .close = vtenc_close,
2603  .capabilities = AV_CODEC_CAP_DELAY,
2604  .priv_class = &h264_videotoolbox_class,
2605  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2606  FF_CODEC_CAP_INIT_CLEANUP,
2607 };
2608 
2609 static const AVOption hevc_options[] = {
2610  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = HEVC_PROF_AUTO }, HEVC_PROF_AUTO, HEVC_PROF_COUNT, VE, "profile" },
2611  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2612  { "main10", "Main10 Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
2613 
2614  COMMON_OPTIONS
2615  { NULL },
2616 };
2617 
2618 static const AVClass hevc_videotoolbox_class = {
2619  .class_name = "hevc_videotoolbox",
2620  .item_name = av_default_item_name,
2621  .option = hevc_options,
2622  .version = LIBAVUTIL_VERSION_INT,
2623 };
2624 
2625 AVCodec ff_hevc_videotoolbox_encoder = {
2626  .name = "hevc_videotoolbox",
2627  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.265 Encoder"),
2628  .type = AVMEDIA_TYPE_VIDEO,
2629  .id = AV_CODEC_ID_HEVC,
2630  .priv_data_size = sizeof(VTEncContext),
2631  .pix_fmts = hevc_pix_fmts,
2632  .init = vtenc_init,
2633  .encode2 = vtenc_frame,
2634  .close = vtenc_close,
2635  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
2636  .priv_class = &hevc_videotoolbox_class,
2637  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2638  FF_CODEC_CAP_INIT_CLEANUP,
2639  .wrapper_name = "videotoolbox",
2640 };
FF_CODEC_CAP_INIT_CLEANUP
#define FF_CODEC_CAP_INIT_CLEANUP
The codec allows calling the close function for deallocation even if the init function returned a fai...
Definition: internal.h:48
AVCOL_TRC_BT2020_12
ITU-R BT2020 for 12-bit system.
Definition: pixfmt.h:499
AVCOL_SPC_BT709
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:514
NULL
#define NULL
Definition: coverity.c:32
AVERROR_INVALIDDATA
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
vtenc_q_push
static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
Definition: videotoolboxenc.c:329
AV_NUM_DATA_POINTERS
#define AV_NUM_DATA_POINTERS
Definition: frame.h:315
get_cm_codec_type
static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
Definition: videotoolboxenc.c:395
pthread_mutex_destroy
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
VTEncContext::cv_sample_sent
pthread_cond_t cv_sample_sent
Definition: videotoolboxenc.c:203
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:314
pthread_mutex_lock
#define pthread_mutex_lock(a)
Definition: ffprobe.c:63
pthread_cond_wait
static av_always_inline int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
Definition: os2threads.h:192
AVOption
AVOption.
Definition: opt.h:248
pthread_once_t
Definition: os2threads.h:66
AVCOL_TRC_LINEAR
"Linear transfer characteristics"
Definition: pixfmt.h:492
VTEncContext::q_head
BufNode * q_head
Definition: videotoolboxenc.c:207
ret
ret
Definition: filter_design.txt:187
BufNode::next
struct BufNode * next
Definition: videotoolboxenc.c:189
AV_LOG_WARNING
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
hevc_videotoolbox_class
static const AVClass hevc_videotoolbox_class
Definition: videotoolboxenc.c:2618
AVCodecContext::bit_rate
int64_t bit_rate
the average bitrate
Definition: avcodec.h:581
LIBAVUTIL_VERSION_INT
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
AV_PIX_FMT_VIDEOTOOLBOX
hardware decoding through Videotoolbox
Definition: pixfmt.h:282
init
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:35
AV_OPT_TYPE_INT
Definition: opt.h:225
set_extradata
static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
Definition: videotoolboxenc.c:530
AVCodecContext::max_b_frames
int max_b_frames
maximum number of B-frames between non-B-frames Note: The output will be delayed by max_b_frames+1 re...
Definition: avcodec.h:791
AVCOL_SPC_BT470BG
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601 ...
Definition: pixfmt.h:518
copy_param_sets
static int copy_param_sets(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, uint8_t *dst, size_t dst_size)
Definition: videotoolboxenc.c:466
AV_CODEC_CAP_HARDWARE
#define AV_CODEC_CAP_HARDWARE
Codec is backed by a hardware implementation.
Definition: codec.h:150
create_cv_pixel_buffer_info
static int create_cv_pixel_buffer_info(AVCodecContext *avctx, CFMutableDictionaryRef *dict)
Definition: videotoolboxenc.c:822
AVCodecContext::color_range
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: avcodec.h:1166
AVRational::num
int num
Numerator.
Definition: rational.h:59
AVPacket::size
int size
Definition: packet.h:364
vtenc_configure_encoder
static int vtenc_configure_encoder(AVCodecContext *avctx)
Definition: videotoolboxenc.c:1319
av_default_item_name
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
write_sei
static int write_sei(const ExtraSEI *sei, int sei_type, uint8_t *dst, size_t dst_size)
Definition: videotoolboxenc.c:1612
AVCodecContext::sample_aspect_ratio
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel...
Definition: avcodec.h:910
AVCOL_SPC_SMPTE170M
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:519
codec_type
enum AVMediaType codec_type
Definition: rtp.c:37
AVCodecContext::pix_fmt
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:741
type
GLint GLenum type
Definition: opengl_enc.c:104
kVTProfileLevel_H264_High_5_2
CFStringRef kVTProfileLevel_H264_High_5_2
Definition: videotoolboxenc.c:82
av_mallocz
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
pkt
static AVPacket pkt
Definition: demuxing_decoding.c:54
AV_CODEC_ID_H264
Definition: codec_id.h:76
AVCodec
AVCodec.
Definition: codec.h:190
pthread_cond_destroy
static av_always_inline int pthread_cond_destroy(pthread_cond_t *cond)
Definition: os2threads.h:144
AVColorTransferCharacteristic
AVColorTransferCharacteristic
Color Transfer Characteristic.
Definition: pixfmt.h:483
AVCOL_SPC_SMPTE240M
functionally identical to above
Definition: pixfmt.h:520
av_color_space_name
const char * av_color_space_name(enum AVColorSpace space)
Definition: pixdesc.c:2966
VTEncContext::require_sw
int64_t require_sw
Definition: videotoolboxenc.c:224
kVTCompressionPropertyKey_H264EntropyMode
CFStringRef kVTCompressionPropertyKey_H264EntropyMode
Definition: videotoolboxenc.c:61
copy_replace_length_codes
static int copy_replace_length_codes(AVCodecContext *avctx, size_t length_code_size, CMSampleBufferRef sample_buffer, ExtraSEI *sei, uint8_t *dst_data, size_t dst_size)
Copies NAL units and replaces length codes with H.264 Annex B start codes.
Definition: videotoolboxenc.c:1685
AVCodecContext::time_base
AVRational time_base
This is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented...
Definition: avcodec.h:654
is_post_sei_nal_type
static int is_post_sei_nal_type(int nal_type)
Definition: videotoolboxenc.c:1486
av_frame_get_side_data
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:738
AVClass::class_name
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
AV_CODEC_CAP_DELAY
#define AV_CODEC_CAP_DELAY
Encoder or decoder requires flushing with NULL input at the end in order to give the complete and cor...
Definition: codec.h:75
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
vtenc_cm_to_avpacket
static int vtenc_cm_to_avpacket(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, AVPacket *pkt, ExtraSEI *sei)
Definition: videotoolboxenc.c:1859
ff_alloc_packet2
int ff_alloc_packet2(AVCodecContext *avctx, AVPacket *avpkt, int64_t size, int64_t min_size)
Check AVPacket size and/or allocate data.
Definition: encode.c:33
AV_OPT_TYPE_CONST
Definition: opt.h:234
ff_hevc_videotoolbox_encoder
AVCodec ff_hevc_videotoolbox_encoder
Definition: videotoolboxenc.c:2625
kCVImageBufferTransferFunction_ITU_R_2020
CFStringRef kCVImageBufferTransferFunction_ITU_R_2020
Definition: videotoolboxenc.c:58
FF_CODEC_CAP_INIT_THREADSAFE
#define FF_CODEC_CAP_INIT_THREADSAFE
The codec does not modify any global variables in the init function, allowing to call the init functi...
Definition: internal.h:40
block
The exact code depends on how similar the blocks are and how related they are to the block
Definition: filter_design.txt:207
uint8_t
uint8_t
Definition: audio_convert.c:194
av_cold
#define av_cold
Definition: attributes.h:88
av_malloc
#define av_malloc(s)
Definition: tableprint_vlc.h:31
kVTProfileLevel_HEVC_Main_AutoLevel
CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel
Definition: videotoolboxenc.c:87
opt.h
AVOptions.
color
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
hevc_options
static const AVOption hevc_options[]
Definition: videotoolboxenc.c:2609
offset
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
Definition: writing_filters.txt:84
av_color_range_name
const char * av_color_range_name(enum AVColorRange range)
Definition: pixdesc.c:2899
AVCOL_TRC_GAMMA22
also ITU-R BT470M / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:488
AVFrame::pts
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:407
kVTProfileLevel_H264_Baseline_5_2
CFStringRef kVTProfileLevel_H264_Baseline_5_2
Definition: videotoolboxenc.c:69
AVCodecContext::extradata
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:632
get_cv_pixel_info
static int get_cv_pixel_info(AVCodecContext *avctx, const AVFrame *frame, int *color, int *plane_count, size_t *widths, size_t *heights, size_t *strides, size_t *contiguous_buf_size)
Definition: videotoolboxenc.c:1969
AVFrameSideData
Structure to hold side data for an AVFrame.
Definition: frame.h:220
pthread_cond_t
Definition: os2threads.h:58
AVPacket::data
uint8_t * data
Definition: packet.h:363
VTEncContext::frame_ct_in
int64_t frame_ct_in
Definition: videotoolboxenc.c:211
AVCOL_RANGE_NB
Not part of ABI.
Definition: pixfmt.h:587
pthread_cond_signal
static av_always_inline int pthread_cond_signal(pthread_cond_t *cond)
Definition: os2threads.h:152
get_length_code_size
static int get_length_code_size(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, size_t *size)
Definition: videotoolboxenc.c:600
kVTProfileLevel_H264_High_4_0
CFStringRef kVTProfileLevel_H264_High_4_0
Definition: videotoolboxenc.c:78
AVColorRange
AVColorRange
Visual content value range.
Definition: pixfmt.h:551
size
ptrdiff_t size
Definition: opengl_enc.c:100
av_reduce
int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max)
Reduce a fraction.
Definition: rational.c:35
ff_alloc_a53_sei
int ff_alloc_a53_sei(const AVFrame *frame, size_t prefix_len, void **data, size_t *sei_size)
Check AVFrame for A53 side data and allocate and fill SEI message with A53 info.
Definition: atsc_a53.c:25
AVColorPrimaries
AVColorPrimaries
Chromaticity coordinates of the source primaries.
Definition: pixfmt.h:458
vtenc_frame
static av_cold int vtenc_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet)
Definition: videotoolboxenc.c:2337
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:28
AV_PKT_FLAG_KEY
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: packet.h:401
kVTProfileLevel_HEVC_Main10_AutoLevel
CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel
Definition: videotoolboxenc.c:88
h264_sei.h
h264.h
H.264 common definitions.
GET_SYM
#define GET_SYM(symbol, defaultVal)
Definition: videotoolboxenc.c:98
src
#define src
Definition: vp8dsp.c:255
kVTProfileLevel_H264_Baseline_5_1
CFStringRef kVTProfileLevel_H264_Baseline_5_1
Definition: videotoolboxenc.c:68
AVCodecID
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:46
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
AVCodecContext::has_b_frames
int has_b_frames
Size of the frame reordering buffer in the decoder.
Definition: avcodec.h:821
kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder
CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder
Definition: videotoolboxenc.c:92
vtenc_get_frame_info
static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
Definition: videotoolboxenc.c:1459
kVTProfileLevel_H264_Baseline_5_0
CFStringRef kVTProfileLevel_H264_Baseline_5_0
Definition: videotoolboxenc.c:67
kVTProfileLevel_H264_Main_4_2
CFStringRef kVTProfileLevel_H264_Main_4_2
Definition: videotoolboxenc.c:71
find_sei_end
static int find_sei_end(AVCodecContext *avctx, uint8_t *nal_data, size_t nal_size, uint8_t **sei_end)
Definition: videotoolboxenc.c:1497
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:115
VTEncContext::q_tail
BufNode * q_tail
Definition: videotoolboxenc.c:208
AVFrame::color_range
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: frame.h:558
AV_FRAME_DATA_A53_CC
ATSC A53 Part 4 Closed Captions.
Definition: frame.h:58
add_color_attr
static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict)
Definition: videotoolboxenc.c:800
kVTProfileLevel_H264_High_4_1
CFStringRef kVTProfileLevel_H264_High_4_1
Definition: videotoolboxenc.c:79
get_params_size
static int get_params_size(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, size_t *size)
Get the parameter sets from a CMSampleBufferRef.
Definition: videotoolboxenc.c:412
AVCodecContext::flags
int flags
AV_CODEC_FLAG_*.
Definition: avcodec.h:611
AV_PIX_FMT_NV12
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
AVCOL_PRI_BT709
also ITU-R BT1361 / IEC 61966-2-4 / SMPTE RP177 Annex B
Definition: pixfmt.h:460
BufNode::sei
ExtraSEI * sei
Definition: videotoolboxenc.c:188
kVTProfileLevel_H264_Extended_5_0
CFStringRef kVTProfileLevel_H264_Extended_5_0
Definition: videotoolboxenc.c:84
avassert.h
simple assert() macros that are a bit more flexible than ISO C assert().
AVCodec::name
const char * name
Name of the codec implementation.
Definition: codec.h:197
kVTProfileLevel_H264_High_3_0
CFStringRef kVTProfileLevel_H264_High_3_0
Definition: videotoolboxenc.c:75
VTH264Entropy
VTH264Entropy
Definition: videotoolboxenc.c:166
VT_HEVCProfile
VT_HEVCProfile
Definition: videotoolboxenc.c:172
kVTProfileLevel_H264_Baseline_4_0
CFStringRef kVTProfileLevel_H264_Baseline_4_0
Definition: videotoolboxenc.c:65
count
GLsizei count
Definition: opengl_enc.c:108
get_frame
static int get_frame(AVFilterContext *ctx, int is_second)
Definition: vf_nnedi.c:689
get_vt_h264_profile_level
static bool get_vt_h264_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
Definition: videotoolboxenc.c:637
AVPacket::flags
int flags
A combination of AV_PKT_FLAG values.
Definition: packet.h:369
av_color_primaries_name
const char * av_color_primaries_name(enum AVColorPrimaries primaries)
Definition: pixdesc.c:2918
VTEncContext::frames_before
int64_t frames_before
Definition: videotoolboxenc.c:220
vtenc_create_encoder
static int vtenc_create_encoder(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info, VTCompressionSessionRef *session)
Definition: videotoolboxenc.c:1017
VTEncContext::session
VTCompressionSessionRef session
Definition: videotoolboxenc.c:196
VTEncContext::color_primaries
CFStringRef color_primaries
Definition: videotoolboxenc.c:198
kVTProfileLevel_H264_High_4_2
CFStringRef kVTProfileLevel_H264_High_4_2
Definition: videotoolboxenc.c:80
info
MIPS optimizations info
Definition: mips.txt:2
get_cv_transfer_function
static int get_cv_transfer_function(AVCodecContext *avctx, CFStringRef *transfer_fnc, CFNumberRef *gamma_level)
Definition: videotoolboxenc.c:923
AVCOL_TRC_SMPTE2084
SMPTE ST 2084 for 10-, 12-, 14- and 16-bit systems.
Definition: pixfmt.h:500
AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
AVFrame::pict_type
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:397
AVERROR_BUFFER_TOO_SMALL
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
BufNode::cm_buffer
CMSampleBufferRef cm_buffer
Definition: videotoolboxenc.c:187
AVCodecContext::width
int width
picture width / height.
Definition: avcodec.h:704
loadVTEncSymbols
static void loadVTEncSymbols()
Definition: videotoolboxenc.c:109
AVCOL_SPC_BT2020_NCL
ITU-R BT2020 non-constant luminance system.
Definition: pixfmt.h:523
AVCOL_PRI_BT470BG
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:465
compat_keys
static struct @168 compat_keys
CMVideoFormatDescriptionGetHEVCParameterSetAtIndex
getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex
Definition: videotoolboxenc.c:95
ctx
AVFormatContext * ctx
Definition: movenc.c:48
AV_OPT_TYPE_BOOL
Definition: opt.h:242
AVCodecContext::color_primaries
enum AVColorPrimaries color_primaries
Chromaticity coordinates of the source primaries.
Definition: avcodec.h:1145
OFFSET
#define OFFSET(x)
Definition: videotoolboxenc.c:2554
pthread_mutex_init
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:255
kVTH264EntropyMode_CAVLC
CFStringRef kVTH264EntropyMode_CAVLC
Definition: videotoolboxenc.c:62
kCVImageBufferYCbCrMatrix_ITU_R_2020
CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020
Definition: videotoolboxenc.c:59
codec_id
enum AVCodecID codec_id
Definition: vaapi_decode.c:369
pthread_mutex_unlock
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:67
AV_PIX_FMT_P010LE
like NV12, with 10bpp per component, data in the high bits, zeros in the low bits, little-endian
Definition: pixfmt.h:284
copy_avframe_to_pixel_buffer
static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef cv_img, const size_t *plane_strides, const size_t *plane_rows)
Definition: videotoolboxenc.c:2076
AVCOL_RANGE_JPEG
Full range content.
Definition: pixfmt.h:586
VTEncContext::codec_id
enum AVCodecID codec_id
Definition: videotoolboxenc.c:195
kVTProfileLevel_H264_Baseline_AutoLevel
CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel
Definition: videotoolboxenc.c:70
vtenc_q_pop
static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
Definition: videotoolboxenc.c:279
AVFrame::format
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:387
kVTCompressionPropertyKey_RealTime
CFStringRef kVTCompressionPropertyKey_RealTime
Definition: videotoolboxenc.c:90
AVCOL_TRC_BT709
also ITU-R BT1361
Definition: pixfmt.h:485
vtenc_populate_extradata
static int vtenc_populate_extradata(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info)
Definition: videotoolboxenc.c:2404
kCVImageBufferColorPrimaries_ITU_R_2020
CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020
Definition: videotoolboxenc.c:57
h264_videotoolbox_class
static const AVClass h264_videotoolbox_class
Definition: videotoolboxenc.c:2586
avcodec.h
Libavcodec external API header.
get_cv_pixel_format
static int get_cv_pixel_format(AVCodecContext *avctx, enum AVPixelFormat fmt, enum AVColorRange range, int *av_pixel_format, int *range_guessed)
Definition: videotoolboxenc.c:771
hevc_pix_fmts
static enum AVPixelFormat hevc_pix_fmts[]
Definition: videotoolboxenc.c:2533
AVCodecContext::codec_id
enum AVCodecID codec_id
Definition: avcodec.h:541
AVFrame::linesize
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:345
kVTProfileLevel_H264_High_3_1
CFStringRef kVTProfileLevel_H264_High_3_1
Definition: videotoolboxenc.c:76
AVCodecContext
main external API structure.
Definition: avcodec.h:531
av_packet_unref
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:606
create_encoder_dict_h264
static int create_encoder_dict_h264(const AVFrame *frame, CFDictionaryRef *dict_out)
Definition: videotoolboxenc.c:2265
vtenc_close
static av_cold int vtenc_close(AVCodecContext *avctx)
Definition: videotoolboxenc.c:2490
AVCodecContext::extradata_size
int extradata_size
Definition: avcodec.h:633
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:67
VTEncContext::transfer_function
CFStringRef transfer_function
Definition: videotoolboxenc.c:199
clear_frame_queue
static void clear_frame_queue(VTEncContext *vtctx)
Definition: videotoolboxenc.c:274
get_cv_ycbcr_matrix
static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix)
Definition: videotoolboxenc.c:986
AVCodecContext::colorspace
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:1159
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
AVCodecContext::color_trc
enum AVColorTransferCharacteristic color_trc
Color Transfer Characteristic.
Definition: avcodec.h:1152
ff_h264_videotoolbox_encoder
AVCodec ff_h264_videotoolbox_encoder
Definition: videotoolboxenc.c:2593
copy_emulation_prev
static int copy_emulation_prev(const uint8_t *src, size_t src_size, uint8_t *dst, ssize_t dst_offset, size_t dst_size)
Copies the data inserting emulation prevention bytes as needed.
Definition: videotoolboxenc.c:1561
vtenc_send_frame
static int vtenc_send_frame(AVCodecContext *avctx, VTEncContext *vtctx, const AVFrame *frame)
Definition: videotoolboxenc.c:2281
status
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
VT_H264Profile
VT_H264Profile
Definition: videotoolboxenc.c:157
once_ctrl
static pthread_once_t once_ctrl
Definition: videotoolboxenc.c:107
H264_SEI_TYPE_USER_DATA_REGISTERED
registered user data as specified by Rec. ITU-T T.35
Definition: h264_sei.h:33
VTEncContext::frames_after
int64_t frames_after
Definition: videotoolboxenc.c:221
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:303
kVTProfileLevel_H264_Main_AutoLevel
CFStringRef kVTProfileLevel_H264_Main_AutoLevel
Definition: videotoolboxenc.c:74
AV_CODEC_ID_HEVC
Definition: codec_id.h:223
profile
mfxU16 profile
Definition: qsvenc.c:45
kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
Definition: videotoolboxenc.c:93
get_sei_msg_bytes
static int get_sei_msg_bytes(const ExtraSEI *sei, int type)
Returns a sufficient number of bytes to contain the sei data.
Definition: videotoolboxenc.c:1842
COMMON_OPTIONS
#define COMMON_OPTIONS
Definition: videotoolboxenc.c:2542
pts
static int64_t pts
Definition: transcode_aac.c:647
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:561
AVFrame::data
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:328
level
uint8_t level
Definition: svq3.c:206
AV_CODEC_FLAG_GLOBAL_HEADER
#define AV_CODEC_FLAG_GLOBAL_HEADER
Place global headers in extradata instead of every keyframe.
Definition: avcodec.h:329
AVCOL_RANGE_MPEG
Narrow or limited range content.
Definition: pixfmt.h:569
AVCodecContext::gop_size
int gop_size
the number of pictures in a group of pictures, or 0 for intra_only
Definition: avcodec.h:726
create_cv_pixel_buffer
static int create_cv_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef *cv_img)
Definition: videotoolboxenc.c:2168
VTEncContext::get_param_set_func
getParameterSetAtIndex get_param_set_func
Definition: videotoolboxenc.c:200
kVTH264EntropyMode_CABAC
CFStringRef kVTH264EntropyMode_CABAC
Definition: videotoolboxenc.c:63
int
int
Definition: ffmpeg_filter.c:193
AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
internal.h
common internal api header.
pthread_mutex_t
_fmutex pthread_mutex_t
Definition: os2threads.h:53
kVTProfileLevel_H264_High_AutoLevel
CFStringRef kVTProfileLevel_H264_High_AutoLevel
Definition: videotoolboxenc.c:83
av_color_transfer_name
const char * av_color_transfer_name(enum AVColorTransferCharacteristic transfer)
Definition: pixdesc.c:2942
getParameterSetAtIndex
OSStatus(* getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc, size_t parameterSetIndex, const uint8_t **parameterSetPointerOut, size_t *parameterSetSizeOut, size_t *parameterSetCountOut, int *NALUnitHeaderLengthOut)
Definition: videotoolboxenc.c:48
VTEncContext::ycbcr_matrix
CFStringRef ycbcr_matrix
Definition: videotoolboxenc.c:197
vtenc_output_callback
static void vtenc_output_callback(void *ctx, void *sourceFrameCtx, OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sample_buffer)
Definition: videotoolboxenc.c:564
AVCOL_TRC_GAMMA28
also ITU-R BT470BG
Definition: pixfmt.h:489
vt_release_num
static void vt_release_num(CFNumberRef *refPtr)
NULL-safe release of *refPtr, and sets value to NULL.
Definition: videotoolboxenc.c:244
kVTProfileLevel_H264_High_3_2
CFStringRef kVTProfileLevel_H264_High_3_2
Definition: videotoolboxenc.c:77
pthread_cond_init
static av_always_inline int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
Definition: os2threads.h:133
VTEncContext::lock
pthread_mutex_t lock
Definition: videotoolboxenc.c:202
sei
static int FUNC() sei(CodedBitstreamContext *ctx, RWContext *rw, H264RawSEI *current)
Definition: cbs_h264_syntax_template.c:924
AVRational::den
int den
Denominator.
Definition: rational.h:60
get_vt_hevc_profile_level
static bool get_vt_hevc_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
Definition: videotoolboxenc.c:742
AV_INPUT_BUFFER_PADDING_SIZE
#define AV_INPUT_BUFFER_PADDING_SIZE
Required number of additionally allocated bytes at the end of the input bitstream for decoding...
Definition: avcodec.h:215
AVCOL_TRC_ARIB_STD_B67
ARIB STD-B67, known as "Hybrid log-gamma".
Definition: pixfmt.h:504
kVTProfileLevel_H264_Extended_AutoLevel
CFStringRef kVTProfileLevel_H264_Extended_AutoLevel
Definition: videotoolboxenc.c:85
AVCodecContext::priv_data
void * priv_data
Definition: avcodec.h:558
PTHREAD_ONCE_INIT
#define PTHREAD_ONCE_INIT
Definition: os2threads.h:71
av_free
#define av_free(p)
Definition: tableprint_vlc.h:34
len
int len
Definition: vorbis_enc_data.h:452
atsc_a53.h
kVTProfileLevel_H264_High_5_1
CFStringRef kVTProfileLevel_H264_High_5_1
Definition: videotoolboxenc.c:81
h264_options
static const AVOption h264_options[]
Definition: videotoolboxenc.c:2555
VE
#define VE
Definition: videotoolboxenc.c:2541
VTEncContext::frame_ct_out
int64_t frame_ct_out
Definition: videotoolboxenc.c:210
kVTProfileLevel_H264_Baseline_4_2
CFStringRef kVTProfileLevel_H264_Baseline_4_2
Definition: videotoolboxenc.c:66
AVCOL_TRC_BT2020_10
ITU-R BT2020 for 10-bit system.
Definition: pixfmt.h:498
AVPacket::dts
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: packet.h:362
AVCOL_PRI_SMPTE170M
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:466
AVCOL_PRI_BT2020
ITU-R BT2020.
Definition: pixfmt.h:469
count_nalus
static int count_nalus(size_t length_code_size, CMSampleBufferRef sample_buffer, int *count)
Definition: videotoolboxenc.c:356
AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
start_code
static const uint8_t start_code[]
Definition: videotoolboxenc.c:179
vtenc_init
static av_cold int vtenc_init(AVCodecContext *avctx)
Definition: videotoolboxenc.c:1429
av_get_pix_fmt_name
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
get_cv_color_primaries
static int get_cv_color_primaries(AVCodecContext *avctx, CFStringRef *primaries)
Definition: videotoolboxenc.c:889
AVERROR_EXTERNAL
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
AVPacket
This structure stores compressed data.
Definition: packet.h:340
pthread_once
static av_always_inline int pthread_once(pthread_once_t *once_control, void(*init_routine)(void))
Definition: os2threads.h:210
set_async_error
static void set_async_error(VTEncContext *vtctx, int err)
Definition: videotoolboxenc.c:253
AVPacket::pts
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:356
i
int i
Definition: input.c:407
buffer
GLuint buffer
Definition: opengl_enc.c:101
kVTProfileLevel_H264_Main_5_1
CFStringRef kVTProfileLevel_H264_Main_5_1
Definition: videotoolboxenc.c:72
kVTProfileLevel_H264_Main_5_2
CFStringRef kVTProfileLevel_H264_Main_5_2
Definition: videotoolboxenc.c:73
AVCodecContext::rc_max_rate
int64_t rc_max_rate
maximum bitrate
Definition: avcodec.h:1409
avc_pix_fmts
static enum AVPixelFormat avc_pix_fmts[]
Definition: videotoolboxenc.c:2526
Generated on Sun Jan 17 2021 19:22:01 for FFmpeg by   doxygen 1.8.11