FFmpeg
videotoolboxenc.c
Go to the documentation of this file.
1 /*
2  * copyright (c) 2015 Rick Kern <kernrj@gmail.com>
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <VideoToolbox/VideoToolbox.h>
22 #include <CoreVideo/CoreVideo.h>
23 #include <CoreMedia/CoreMedia.h>
24 #include <TargetConditionals.h>
25 #include <Availability.h>
26 #include "avcodec.h"
27 #include "libavutil/opt.h"
28 #include "libavutil/avassert.h"
29 #include "libavutil/avstring.h"
30 #include "libavcodec/avcodec.h"
31 #include "libavutil/pixdesc.h"
32 #include "internal.h"
33 #include <pthread.h>
34 #include "atsc_a53.h"
35 #include "h264.h"
36 #include "h264_sei.h"
37 #include <dlfcn.h>
38 
39 #if !HAVE_KCMVIDEOCODECTYPE_HEVC
40 enum { kCMVideoCodecType_HEVC = 'hvc1' };
41 #endif
42 
43 #if !HAVE_KCVPIXELFORMATTYPE_420YPCBCR10BIPLANARVIDEORANGE
46 #endif
47 
48 typedef OSStatus (*getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc,
49  size_t parameterSetIndex,
50  const uint8_t **parameterSetPointerOut,
51  size_t *parameterSetSizeOut,
52  size_t *parameterSetCountOut,
53  int *NALUnitHeaderLengthOut);
54 
55 //These symbols may not be present
56 static struct{
60 
64 
86 
89 
91 
94 
96 } compat_keys;
97 
98 #define GET_SYM(symbol, defaultVal) \
99 do{ \
100  CFStringRef* handle = (CFStringRef*)dlsym(RTLD_DEFAULT, #symbol); \
101  if(!handle) \
102  compat_keys.symbol = CFSTR(defaultVal); \
103  else \
104  compat_keys.symbol = *handle; \
105 }while(0)
106 
108 
109 static void loadVTEncSymbols(){
110  compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex =
111  (getParameterSetAtIndex)dlsym(
112  RTLD_DEFAULT,
113  "CMVideoFormatDescriptionGetHEVCParameterSetAtIndex"
114  );
115 
119 
123 
124  GET_SYM(kVTProfileLevel_H264_Baseline_4_0, "H264_Baseline_4_0");
125  GET_SYM(kVTProfileLevel_H264_Baseline_4_2, "H264_Baseline_4_2");
126  GET_SYM(kVTProfileLevel_H264_Baseline_5_0, "H264_Baseline_5_0");
127  GET_SYM(kVTProfileLevel_H264_Baseline_5_1, "H264_Baseline_5_1");
128  GET_SYM(kVTProfileLevel_H264_Baseline_5_2, "H264_Baseline_5_2");
129  GET_SYM(kVTProfileLevel_H264_Baseline_AutoLevel, "H264_Baseline_AutoLevel");
130  GET_SYM(kVTProfileLevel_H264_Main_4_2, "H264_Main_4_2");
131  GET_SYM(kVTProfileLevel_H264_Main_5_1, "H264_Main_5_1");
132  GET_SYM(kVTProfileLevel_H264_Main_5_2, "H264_Main_5_2");
133  GET_SYM(kVTProfileLevel_H264_Main_AutoLevel, "H264_Main_AutoLevel");
134  GET_SYM(kVTProfileLevel_H264_High_3_0, "H264_High_3_0");
135  GET_SYM(kVTProfileLevel_H264_High_3_1, "H264_High_3_1");
136  GET_SYM(kVTProfileLevel_H264_High_3_2, "H264_High_3_2");
137  GET_SYM(kVTProfileLevel_H264_High_4_0, "H264_High_4_0");
138  GET_SYM(kVTProfileLevel_H264_High_4_1, "H264_High_4_1");
139  GET_SYM(kVTProfileLevel_H264_High_4_2, "H264_High_4_2");
140  GET_SYM(kVTProfileLevel_H264_High_5_1, "H264_High_5_1");
141  GET_SYM(kVTProfileLevel_H264_High_5_2, "H264_High_5_2");
142  GET_SYM(kVTProfileLevel_H264_High_AutoLevel, "H264_High_AutoLevel");
143  GET_SYM(kVTProfileLevel_H264_Extended_5_0, "H264_Extended_5_0");
144  GET_SYM(kVTProfileLevel_H264_Extended_AutoLevel, "H264_Extended_AutoLevel");
145 
146  GET_SYM(kVTProfileLevel_HEVC_Main_AutoLevel, "HEVC_Main_AutoLevel");
147  GET_SYM(kVTProfileLevel_HEVC_Main10_AutoLevel, "HEVC_Main10_AutoLevel");
148 
150 
152  "EnableHardwareAcceleratedVideoEncoder");
154  "RequireHardwareAcceleratedVideoEncoder");
155 }
156 
157 typedef enum VT_H264Profile {
165 
166 typedef enum VTH264Entropy{
170 } VTH264Entropy;
171 
172 typedef enum VT_HEVCProfile {
178 
179 static const uint8_t start_code[] = { 0, 0, 0, 1 };
180 
181 typedef struct ExtraSEI {
182  void *data;
183  size_t size;
184 } ExtraSEI;
185 
186 typedef struct BufNode {
187  CMSampleBufferRef cm_buffer;
189  struct BufNode* next;
190  int error;
191 } BufNode;
192 
193 typedef struct VTEncContext {
194  AVClass *class;
196  VTCompressionSessionRef session;
197  CFStringRef ycbcr_matrix;
198  CFStringRef color_primaries;
199  CFStringRef transfer_function;
201 
204 
206 
209 
210  int64_t frame_ct_out;
211  int64_t frame_ct_in;
212 
213  int64_t first_pts;
214  int64_t dts_delta;
215 
216  int64_t profile;
217  int64_t level;
218  int64_t entropy;
219  int64_t realtime;
220  int64_t frames_before;
221  int64_t frames_after;
222 
223  int64_t allow_sw;
224  int64_t require_sw;
225 
226  bool flushing;
229 
230  /* can't be bool type since AVOption will access it as int */
231  int a53_cc;
232 } VTEncContext;
233 
234 static int vtenc_populate_extradata(AVCodecContext *avctx,
235  CMVideoCodecType codec_type,
236  CFStringRef profile_level,
237  CFNumberRef gamma_level,
238  CFDictionaryRef enc_info,
239  CFDictionaryRef pixel_buffer_info);
240 
241 /**
242  * NULL-safe release of *refPtr, and sets value to NULL.
243  */
244 static void vt_release_num(CFNumberRef* refPtr){
245  if (!*refPtr) {
246  return;
247  }
248 
249  CFRelease(*refPtr);
250  *refPtr = NULL;
251 }
252 
253 static void set_async_error(VTEncContext *vtctx, int err)
254 {
255  BufNode *info;
256 
257  pthread_mutex_lock(&vtctx->lock);
258 
259  vtctx->async_error = err;
260 
261  info = vtctx->q_head;
262  vtctx->q_head = vtctx->q_tail = NULL;
263 
264  while (info) {
265  BufNode *next = info->next;
266  CFRelease(info->cm_buffer);
267  av_free(info);
268  info = next;
269  }
270 
271  pthread_mutex_unlock(&vtctx->lock);
272 }
273 
274 static void clear_frame_queue(VTEncContext *vtctx)
275 {
276  set_async_error(vtctx, 0);
277 }
278 
279 static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
280 {
281  BufNode *info;
282 
283  pthread_mutex_lock(&vtctx->lock);
284 
285  if (vtctx->async_error) {
286  pthread_mutex_unlock(&vtctx->lock);
287  return vtctx->async_error;
288  }
289 
290  if (vtctx->flushing && vtctx->frame_ct_in == vtctx->frame_ct_out) {
291  *buf = NULL;
292 
293  pthread_mutex_unlock(&vtctx->lock);
294  return 0;
295  }
296 
297  while (!vtctx->q_head && !vtctx->async_error && wait && !vtctx->flushing) {
298  pthread_cond_wait(&vtctx->cv_sample_sent, &vtctx->lock);
299  }
300 
301  if (!vtctx->q_head) {
302  pthread_mutex_unlock(&vtctx->lock);
303  *buf = NULL;
304  return 0;
305  }
306 
307  info = vtctx->q_head;
308  vtctx->q_head = vtctx->q_head->next;
309  if (!vtctx->q_head) {
310  vtctx->q_tail = NULL;
311  }
312 
313  vtctx->frame_ct_out++;
314  pthread_mutex_unlock(&vtctx->lock);
315 
316  *buf = info->cm_buffer;
317  if (sei && *buf) {
318  *sei = info->sei;
319  } else if (info->sei) {
320  if (info->sei->data) av_free(info->sei->data);
321  av_free(info->sei);
322  }
323  av_free(info);
324 
325 
326  return 0;
327 }
328 
329 static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
330 {
331  BufNode *info = av_malloc(sizeof(BufNode));
332  if (!info) {
333  set_async_error(vtctx, AVERROR(ENOMEM));
334  return;
335  }
336 
337  CFRetain(buffer);
338  info->cm_buffer = buffer;
339  info->sei = sei;
340  info->next = NULL;
341 
342  pthread_mutex_lock(&vtctx->lock);
343 
344  if (!vtctx->q_head) {
345  vtctx->q_head = info;
346  } else {
347  vtctx->q_tail->next = info;
348  }
349 
350  vtctx->q_tail = info;
351 
353  pthread_mutex_unlock(&vtctx->lock);
354 }
355 
356 static int count_nalus(size_t length_code_size,
357  CMSampleBufferRef sample_buffer,
358  int *count)
359 {
360  size_t offset = 0;
361  int status;
362  int nalu_ct = 0;
363  uint8_t size_buf[4];
364  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
365  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
366 
367  if (length_code_size > 4)
368  return AVERROR_INVALIDDATA;
369 
370  while (offset < src_size) {
371  size_t curr_src_len;
372  size_t box_len = 0;
373  size_t i;
374 
375  status = CMBlockBufferCopyDataBytes(block,
376  offset,
377  length_code_size,
378  size_buf);
379 
380  for (i = 0; i < length_code_size; i++) {
381  box_len <<= 8;
382  box_len |= size_buf[i];
383  }
384 
385  curr_src_len = box_len + length_code_size;
386  offset += curr_src_len;
387 
388  nalu_ct++;
389  }
390 
391  *count = nalu_ct;
392  return 0;
393 }
394 
395 static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
396 {
397  switch (id) {
398  case AV_CODEC_ID_H264: return kCMVideoCodecType_H264;
400  default: return 0;
401  }
402 }
403 
404 /**
405  * Get the parameter sets from a CMSampleBufferRef.
406  * @param dst If *dst isn't NULL, the parameters are copied into existing
407  * memory. *dst_size must be set accordingly when *dst != NULL.
408  * If *dst is NULL, it will be allocated.
409  * In all cases, *dst_size is set to the number of bytes used starting
410  * at *dst.
411  */
412 static int get_params_size(
413  AVCodecContext *avctx,
414  CMVideoFormatDescriptionRef vid_fmt,
415  size_t *size)
416 {
417  VTEncContext *vtctx = avctx->priv_data;
418  size_t total_size = 0;
419  size_t ps_count;
420  int is_count_bad = 0;
421  size_t i;
422  int status;
423  status = vtctx->get_param_set_func(vid_fmt,
424  0,
425  NULL,
426  NULL,
427  &ps_count,
428  NULL);
429  if (status) {
430  is_count_bad = 1;
431  ps_count = 0;
432  status = 0;
433  }
434 
435  for (i = 0; i < ps_count || is_count_bad; i++) {
436  const uint8_t *ps;
437  size_t ps_size;
438  status = vtctx->get_param_set_func(vid_fmt,
439  i,
440  &ps,
441  &ps_size,
442  NULL,
443  NULL);
444  if (status) {
445  /*
446  * When ps_count is invalid, status != 0 ends the loop normally
447  * unless we didn't get any parameter sets.
448  */
449  if (i > 0 && is_count_bad) status = 0;
450 
451  break;
452  }
453 
454  total_size += ps_size + sizeof(start_code);
455  }
456 
457  if (status) {
458  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set sizes: %d\n", status);
459  return AVERROR_EXTERNAL;
460  }
461 
462  *size = total_size;
463  return 0;
464 }
465 
466 static int copy_param_sets(
467  AVCodecContext *avctx,
468  CMVideoFormatDescriptionRef vid_fmt,
469  uint8_t *dst,
470  size_t dst_size)
471 {
472  VTEncContext *vtctx = avctx->priv_data;
473  size_t ps_count;
474  int is_count_bad = 0;
475  int status;
476  size_t offset = 0;
477  size_t i;
478 
479  status = vtctx->get_param_set_func(vid_fmt,
480  0,
481  NULL,
482  NULL,
483  &ps_count,
484  NULL);
485  if (status) {
486  is_count_bad = 1;
487  ps_count = 0;
488  status = 0;
489  }
490 
491 
492  for (i = 0; i < ps_count || is_count_bad; i++) {
493  const uint8_t *ps;
494  size_t ps_size;
495  size_t next_offset;
496 
497  status = vtctx->get_param_set_func(vid_fmt,
498  i,
499  &ps,
500  &ps_size,
501  NULL,
502  NULL);
503  if (status) {
504  if (i > 0 && is_count_bad) status = 0;
505 
506  break;
507  }
508 
509  next_offset = offset + sizeof(start_code) + ps_size;
510  if (dst_size < next_offset) {
511  av_log(avctx, AV_LOG_ERROR, "Error: buffer too small for parameter sets.\n");
513  }
514 
515  memcpy(dst + offset, start_code, sizeof(start_code));
516  offset += sizeof(start_code);
517 
518  memcpy(dst + offset, ps, ps_size);
519  offset = next_offset;
520  }
521 
522  if (status) {
523  av_log(avctx, AV_LOG_ERROR, "Error getting parameter set data: %d\n", status);
524  return AVERROR_EXTERNAL;
525  }
526 
527  return 0;
528 }
529 
530 static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
531 {
532  CMVideoFormatDescriptionRef vid_fmt;
533  size_t total_size;
534  int status;
535 
536  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
537  if (!vid_fmt) {
538  av_log(avctx, AV_LOG_ERROR, "No video format.\n");
539  return AVERROR_EXTERNAL;
540  }
541 
542  status = get_params_size(avctx, vid_fmt, &total_size);
543  if (status) {
544  av_log(avctx, AV_LOG_ERROR, "Could not get parameter sets.\n");
545  return status;
546  }
547 
548  avctx->extradata = av_mallocz(total_size + AV_INPUT_BUFFER_PADDING_SIZE);
549  if (!avctx->extradata) {
550  return AVERROR(ENOMEM);
551  }
552  avctx->extradata_size = total_size;
553 
554  status = copy_param_sets(avctx, vid_fmt, avctx->extradata, total_size);
555 
556  if (status) {
557  av_log(avctx, AV_LOG_ERROR, "Could not copy param sets.\n");
558  return status;
559  }
560 
561  return 0;
562 }
563 
565  void *ctx,
566  void *sourceFrameCtx,
567  OSStatus status,
568  VTEncodeInfoFlags flags,
569  CMSampleBufferRef sample_buffer)
570 {
571  AVCodecContext *avctx = ctx;
572  VTEncContext *vtctx = avctx->priv_data;
573  ExtraSEI *sei = sourceFrameCtx;
574 
575  if (vtctx->async_error) {
576  return;
577  }
578 
579  if (status) {
580  av_log(avctx, AV_LOG_ERROR, "Error encoding frame: %d\n", (int)status);
582  return;
583  }
584 
585  if (!sample_buffer) {
586  return;
587  }
588 
589  if (!avctx->extradata && (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER)) {
590  int set_status = set_extradata(avctx, sample_buffer);
591  if (set_status) {
592  set_async_error(vtctx, set_status);
593  return;
594  }
595  }
596 
597  vtenc_q_push(vtctx, sample_buffer, sei);
598 }
599 
601  AVCodecContext *avctx,
602  CMSampleBufferRef sample_buffer,
603  size_t *size)
604 {
605  VTEncContext *vtctx = avctx->priv_data;
606  CMVideoFormatDescriptionRef vid_fmt;
607  int isize;
608  int status;
609 
610  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
611  if (!vid_fmt) {
612  av_log(avctx, AV_LOG_ERROR, "Error getting buffer format description.\n");
613  return AVERROR_EXTERNAL;
614  }
615 
616  status = vtctx->get_param_set_func(vid_fmt,
617  0,
618  NULL,
619  NULL,
620  NULL,
621  &isize);
622  if (status) {
623  av_log(avctx, AV_LOG_ERROR, "Error getting length code size: %d\n", status);
624  return AVERROR_EXTERNAL;
625  }
626 
627  *size = isize;
628  return 0;
629 }
630 
631 /*
632  * Returns true on success.
633  *
634  * If profile_level_val is NULL and this method returns true, don't specify the
635  * profile/level to the encoder.
636  */
638  CFStringRef *profile_level_val)
639 {
640  VTEncContext *vtctx = avctx->priv_data;
641  int64_t profile = vtctx->profile;
642 
643  if (profile == H264_PROF_AUTO && vtctx->level) {
644  //Need to pick a profile if level is not auto-selected.
645  profile = vtctx->has_b_frames ? H264_PROF_MAIN : H264_PROF_BASELINE;
646  }
647 
648  *profile_level_val = NULL;
649 
650  switch (profile) {
651  case H264_PROF_AUTO:
652  return true;
653 
654  case H264_PROF_BASELINE:
655  switch (vtctx->level) {
656  case 0: *profile_level_val =
657  compat_keys.kVTProfileLevel_H264_Baseline_AutoLevel; break;
658  case 13: *profile_level_val = kVTProfileLevel_H264_Baseline_1_3; break;
659  case 30: *profile_level_val = kVTProfileLevel_H264_Baseline_3_0; break;
660  case 31: *profile_level_val = kVTProfileLevel_H264_Baseline_3_1; break;
661  case 32: *profile_level_val = kVTProfileLevel_H264_Baseline_3_2; break;
662  case 40: *profile_level_val =
663  compat_keys.kVTProfileLevel_H264_Baseline_4_0; break;
664  case 41: *profile_level_val = kVTProfileLevel_H264_Baseline_4_1; break;
665  case 42: *profile_level_val =
666  compat_keys.kVTProfileLevel_H264_Baseline_4_2; break;
667  case 50: *profile_level_val =
668  compat_keys.kVTProfileLevel_H264_Baseline_5_0; break;
669  case 51: *profile_level_val =
670  compat_keys.kVTProfileLevel_H264_Baseline_5_1; break;
671  case 52: *profile_level_val =
672  compat_keys.kVTProfileLevel_H264_Baseline_5_2; break;
673  }
674  break;
675 
676  case H264_PROF_MAIN:
677  switch (vtctx->level) {
678  case 0: *profile_level_val =
679  compat_keys.kVTProfileLevel_H264_Main_AutoLevel; break;
680  case 30: *profile_level_val = kVTProfileLevel_H264_Main_3_0; break;
681  case 31: *profile_level_val = kVTProfileLevel_H264_Main_3_1; break;
682  case 32: *profile_level_val = kVTProfileLevel_H264_Main_3_2; break;
683  case 40: *profile_level_val = kVTProfileLevel_H264_Main_4_0; break;
684  case 41: *profile_level_val = kVTProfileLevel_H264_Main_4_1; break;
685  case 42: *profile_level_val =
686  compat_keys.kVTProfileLevel_H264_Main_4_2; break;
687  case 50: *profile_level_val = kVTProfileLevel_H264_Main_5_0; break;
688  case 51: *profile_level_val =
689  compat_keys.kVTProfileLevel_H264_Main_5_1; break;
690  case 52: *profile_level_val =
691  compat_keys.kVTProfileLevel_H264_Main_5_2; break;
692  }
693  break;
694 
695  case H264_PROF_HIGH:
696  switch (vtctx->level) {
697  case 0: *profile_level_val =
698  compat_keys.kVTProfileLevel_H264_High_AutoLevel; break;
699  case 30: *profile_level_val =
700  compat_keys.kVTProfileLevel_H264_High_3_0; break;
701  case 31: *profile_level_val =
702  compat_keys.kVTProfileLevel_H264_High_3_1; break;
703  case 32: *profile_level_val =
704  compat_keys.kVTProfileLevel_H264_High_3_2; break;
705  case 40: *profile_level_val =
706  compat_keys.kVTProfileLevel_H264_High_4_0; break;
707  case 41: *profile_level_val =
708  compat_keys.kVTProfileLevel_H264_High_4_1; break;
709  case 42: *profile_level_val =
710  compat_keys.kVTProfileLevel_H264_High_4_2; break;
711  case 50: *profile_level_val = kVTProfileLevel_H264_High_5_0; break;
712  case 51: *profile_level_val =
713  compat_keys.kVTProfileLevel_H264_High_5_1; break;
714  case 52: *profile_level_val =
715  compat_keys.kVTProfileLevel_H264_High_5_2; break;
716  }
717  break;
718  case H264_PROF_EXTENDED:
719  switch (vtctx->level) {
720  case 0: *profile_level_val =
721  compat_keys.kVTProfileLevel_H264_Extended_AutoLevel; break;
722  case 50: *profile_level_val =
723  compat_keys.kVTProfileLevel_H264_Extended_5_0; break;
724  }
725  break;
726  }
727 
728  if (!*profile_level_val) {
729  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
730  return false;
731  }
732 
733  return true;
734 }
735 
736 /*
737  * Returns true on success.
738  *
739  * If profile_level_val is NULL and this method returns true, don't specify the
740  * profile/level to the encoder.
741  */
743  CFStringRef *profile_level_val)
744 {
745  VTEncContext *vtctx = avctx->priv_data;
746  int64_t profile = vtctx->profile;
747 
748  *profile_level_val = NULL;
749 
750  switch (profile) {
751  case HEVC_PROF_AUTO:
752  return true;
753  case HEVC_PROF_MAIN:
754  *profile_level_val =
755  compat_keys.kVTProfileLevel_HEVC_Main_AutoLevel;
756  break;
757  case HEVC_PROF_MAIN10:
758  *profile_level_val =
759  compat_keys.kVTProfileLevel_HEVC_Main10_AutoLevel;
760  break;
761  }
762 
763  if (!*profile_level_val) {
764  av_log(avctx, AV_LOG_ERROR, "Invalid Profile/Level.\n");
765  return false;
766  }
767 
768  return true;
769 }
770 
772  enum AVPixelFormat fmt,
773  enum AVColorRange range,
774  int* av_pixel_format,
775  int* range_guessed)
776 {
777  if (range_guessed) *range_guessed = range != AVCOL_RANGE_MPEG &&
778  range != AVCOL_RANGE_JPEG;
779 
780  //MPEG range is used when no range is set
781  if (fmt == AV_PIX_FMT_NV12) {
782  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
783  kCVPixelFormatType_420YpCbCr8BiPlanarFullRange :
784  kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
785  } else if (fmt == AV_PIX_FMT_YUV420P) {
786  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
787  kCVPixelFormatType_420YpCbCr8PlanarFullRange :
788  kCVPixelFormatType_420YpCbCr8Planar;
789  } else if (fmt == AV_PIX_FMT_P010LE) {
790  *av_pixel_format = range == AVCOL_RANGE_JPEG ?
793  } else {
794  return AVERROR(EINVAL);
795  }
796 
797  return 0;
798 }
799 
800 static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict) {
801  VTEncContext *vtctx = avctx->priv_data;
802 
803  if (vtctx->color_primaries) {
804  CFDictionarySetValue(dict,
805  kCVImageBufferColorPrimariesKey,
806  vtctx->color_primaries);
807  }
808 
809  if (vtctx->transfer_function) {
810  CFDictionarySetValue(dict,
811  kCVImageBufferTransferFunctionKey,
812  vtctx->transfer_function);
813  }
814 
815  if (vtctx->ycbcr_matrix) {
816  CFDictionarySetValue(dict,
817  kCVImageBufferYCbCrMatrixKey,
818  vtctx->ycbcr_matrix);
819  }
820 }
821 
823  CFMutableDictionaryRef* dict)
824 {
825  CFNumberRef cv_color_format_num = NULL;
826  CFNumberRef width_num = NULL;
827  CFNumberRef height_num = NULL;
828  CFMutableDictionaryRef pixel_buffer_info = NULL;
829  int cv_color_format;
830  int status = get_cv_pixel_format(avctx,
831  avctx->pix_fmt,
832  avctx->color_range,
833  &cv_color_format,
834  NULL);
835  if (status) return status;
836 
837  pixel_buffer_info = CFDictionaryCreateMutable(
838  kCFAllocatorDefault,
839  20,
840  &kCFCopyStringDictionaryKeyCallBacks,
841  &kCFTypeDictionaryValueCallBacks);
842 
843  if (!pixel_buffer_info) goto pbinfo_nomem;
844 
845  cv_color_format_num = CFNumberCreate(kCFAllocatorDefault,
846  kCFNumberSInt32Type,
847  &cv_color_format);
848  if (!cv_color_format_num) goto pbinfo_nomem;
849 
850  CFDictionarySetValue(pixel_buffer_info,
851  kCVPixelBufferPixelFormatTypeKey,
852  cv_color_format_num);
853  vt_release_num(&cv_color_format_num);
854 
855  width_num = CFNumberCreate(kCFAllocatorDefault,
856  kCFNumberSInt32Type,
857  &avctx->width);
858  if (!width_num) return AVERROR(ENOMEM);
859 
860  CFDictionarySetValue(pixel_buffer_info,
861  kCVPixelBufferWidthKey,
862  width_num);
863  vt_release_num(&width_num);
864 
865  height_num = CFNumberCreate(kCFAllocatorDefault,
866  kCFNumberSInt32Type,
867  &avctx->height);
868  if (!height_num) goto pbinfo_nomem;
869 
870  CFDictionarySetValue(pixel_buffer_info,
871  kCVPixelBufferHeightKey,
872  height_num);
873  vt_release_num(&height_num);
874 
875  add_color_attr(avctx, pixel_buffer_info);
876 
877  *dict = pixel_buffer_info;
878  return 0;
879 
880 pbinfo_nomem:
881  vt_release_num(&cv_color_format_num);
882  vt_release_num(&width_num);
883  vt_release_num(&height_num);
884  if (pixel_buffer_info) CFRelease(pixel_buffer_info);
885 
886  return AVERROR(ENOMEM);
887 }
888 
890  CFStringRef *primaries)
891 {
892  enum AVColorPrimaries pri = avctx->color_primaries;
893  switch (pri) {
895  *primaries = NULL;
896  break;
897 
898  case AVCOL_PRI_BT470BG:
899  *primaries = kCVImageBufferColorPrimaries_EBU_3213;
900  break;
901 
902  case AVCOL_PRI_SMPTE170M:
903  *primaries = kCVImageBufferColorPrimaries_SMPTE_C;
904  break;
905 
906  case AVCOL_PRI_BT709:
907  *primaries = kCVImageBufferColorPrimaries_ITU_R_709_2;
908  break;
909 
910  case AVCOL_PRI_BT2020:
911  *primaries = compat_keys.kCVImageBufferColorPrimaries_ITU_R_2020;
912  break;
913 
914  default:
915  av_log(avctx, AV_LOG_ERROR, "Color primaries %s is not supported.\n", av_color_primaries_name(pri));
916  *primaries = NULL;
917  return -1;
918  }
919 
920  return 0;
921 }
922 
924  CFStringRef *transfer_fnc,
925  CFNumberRef *gamma_level)
926 {
927  enum AVColorTransferCharacteristic trc = avctx->color_trc;
928  Float32 gamma;
929  *gamma_level = NULL;
930 
931  switch (trc) {
933  *transfer_fnc = NULL;
934  break;
935 
936  case AVCOL_TRC_BT709:
937  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_709_2;
938  break;
939 
940  case AVCOL_TRC_SMPTE240M:
941  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_240M_1995;
942  break;
943 
944 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_SMPTE_ST_2084_PQ
945  case AVCOL_TRC_SMPTE2084:
946  *transfer_fnc = kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ;
947  break;
948 #endif
949 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_LINEAR
950  case AVCOL_TRC_LINEAR:
951  *transfer_fnc = kCVImageBufferTransferFunction_Linear;
952  break;
953 #endif
954 #if HAVE_KCVIMAGEBUFFERTRANSFERFUNCTION_ITU_R_2100_HLG
956  *transfer_fnc = kCVImageBufferTransferFunction_ITU_R_2100_HLG;
957  break;
958 #endif
959 
960  case AVCOL_TRC_GAMMA22:
961  gamma = 2.2;
962  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
963  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
964  break;
965 
966  case AVCOL_TRC_GAMMA28:
967  gamma = 2.8;
968  *transfer_fnc = kCVImageBufferTransferFunction_UseGamma;
969  *gamma_level = CFNumberCreate(NULL, kCFNumberFloat32Type, &gamma);
970  break;
971 
972  case AVCOL_TRC_BT2020_10:
973  case AVCOL_TRC_BT2020_12:
974  *transfer_fnc = compat_keys.kCVImageBufferTransferFunction_ITU_R_2020;
975  break;
976 
977  default:
978  *transfer_fnc = NULL;
979  av_log(avctx, AV_LOG_ERROR, "Transfer function %s is not supported.\n", av_color_transfer_name(trc));
980  return -1;
981  }
982 
983  return 0;
984 }
985 
986 static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix) {
987  switch(avctx->colorspace) {
988  case AVCOL_SPC_BT709:
989  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_709_2;
990  break;
991 
993  *matrix = NULL;
994  break;
995 
996  case AVCOL_SPC_BT470BG:
997  case AVCOL_SPC_SMPTE170M:
998  *matrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
999  break;
1000 
1001  case AVCOL_SPC_SMPTE240M:
1002  *matrix = kCVImageBufferYCbCrMatrix_SMPTE_240M_1995;
1003  break;
1004 
1005  case AVCOL_SPC_BT2020_NCL:
1006  *matrix = compat_keys.kCVImageBufferYCbCrMatrix_ITU_R_2020;
1007  break;
1008 
1009  default:
1010  av_log(avctx, AV_LOG_ERROR, "Color space %s is not supported.\n", av_color_space_name(avctx->colorspace));
1011  return -1;
1012  }
1013 
1014  return 0;
1015 }
1016 
1018  CMVideoCodecType codec_type,
1019  CFStringRef profile_level,
1020  CFNumberRef gamma_level,
1021  CFDictionaryRef enc_info,
1022  CFDictionaryRef pixel_buffer_info,
1023  VTCompressionSessionRef *session)
1024 {
1025  VTEncContext *vtctx = avctx->priv_data;
1026  SInt32 bit_rate = avctx->bit_rate;
1027  SInt32 max_rate = avctx->rc_max_rate;
1028  CFNumberRef bit_rate_num;
1029  CFNumberRef bytes_per_second;
1030  CFNumberRef one_second;
1031  CFArrayRef data_rate_limits;
1032  int64_t bytes_per_second_value = 0;
1033  int64_t one_second_value = 0;
1034  void *nums[2];
1035 
1036  int status = VTCompressionSessionCreate(kCFAllocatorDefault,
1037  avctx->width,
1038  avctx->height,
1039  codec_type,
1040  enc_info,
1041  pixel_buffer_info,
1042  kCFAllocatorDefault,
1044  avctx,
1045  session);
1046 
1047  if (status || !vtctx->session) {
1048  av_log(avctx, AV_LOG_ERROR, "Error: cannot create compression session: %d\n", status);
1049 
1050 #if !TARGET_OS_IPHONE
1051  if (!vtctx->allow_sw) {
1052  av_log(avctx, AV_LOG_ERROR, "Try -allow_sw 1. The hardware encoder may be busy, or not supported.\n");
1053  }
1054 #endif
1055 
1056  return AVERROR_EXTERNAL;
1057  }
1058 
1059  bit_rate_num = CFNumberCreate(kCFAllocatorDefault,
1060  kCFNumberSInt32Type,
1061  &bit_rate);
1062  if (!bit_rate_num) return AVERROR(ENOMEM);
1063 
1064  status = VTSessionSetProperty(vtctx->session,
1065  kVTCompressionPropertyKey_AverageBitRate,
1066  bit_rate_num);
1067  CFRelease(bit_rate_num);
1068 
1069  if (status) {
1070  av_log(avctx, AV_LOG_ERROR, "Error setting bitrate property: %d\n", status);
1071  return AVERROR_EXTERNAL;
1072  }
1073 
1074  if (vtctx->codec_id == AV_CODEC_ID_H264 && max_rate > 0) {
1075  // kVTCompressionPropertyKey_DataRateLimits is not available for HEVC
1076  bytes_per_second_value = max_rate >> 3;
1077  bytes_per_second = CFNumberCreate(kCFAllocatorDefault,
1078  kCFNumberSInt64Type,
1079  &bytes_per_second_value);
1080  if (!bytes_per_second) {
1081  return AVERROR(ENOMEM);
1082  }
1083  one_second_value = 1;
1084  one_second = CFNumberCreate(kCFAllocatorDefault,
1085  kCFNumberSInt64Type,
1086  &one_second_value);
1087  if (!one_second) {
1088  CFRelease(bytes_per_second);
1089  return AVERROR(ENOMEM);
1090  }
1091  nums[0] = (void *)bytes_per_second;
1092  nums[1] = (void *)one_second;
1093  data_rate_limits = CFArrayCreate(kCFAllocatorDefault,
1094  (const void **)nums,
1095  2,
1096  &kCFTypeArrayCallBacks);
1097 
1098  if (!data_rate_limits) {
1099  CFRelease(bytes_per_second);
1100  CFRelease(one_second);
1101  return AVERROR(ENOMEM);
1102  }
1103  status = VTSessionSetProperty(vtctx->session,
1104  kVTCompressionPropertyKey_DataRateLimits,
1105  data_rate_limits);
1106 
1107  CFRelease(bytes_per_second);
1108  CFRelease(one_second);
1109  CFRelease(data_rate_limits);
1110 
1111  if (status) {
1112  av_log(avctx, AV_LOG_ERROR, "Error setting max bitrate property: %d\n", status);
1113  return AVERROR_EXTERNAL;
1114  }
1115  }
1116 
1117  if (profile_level) {
1118  status = VTSessionSetProperty(vtctx->session,
1119  kVTCompressionPropertyKey_ProfileLevel,
1120  profile_level);
1121  if (status) {
1122  av_log(avctx, AV_LOG_ERROR, "Error setting profile/level property: %d. Output will be encoded using a supported profile/level combination.\n", status);
1123  }
1124  }
1125 
1126  if (avctx->gop_size > 0) {
1127  CFNumberRef interval = CFNumberCreate(kCFAllocatorDefault,
1128  kCFNumberIntType,
1129  &avctx->gop_size);
1130  if (!interval) {
1131  return AVERROR(ENOMEM);
1132  }
1133 
1134  status = VTSessionSetProperty(vtctx->session,
1135  kVTCompressionPropertyKey_MaxKeyFrameInterval,
1136  interval);
1137  CFRelease(interval);
1138 
1139  if (status) {
1140  av_log(avctx, AV_LOG_ERROR, "Error setting 'max key-frame interval' property: %d\n", status);
1141  return AVERROR_EXTERNAL;
1142  }
1143  }
1144 
1145  if (vtctx->frames_before) {
1146  status = VTSessionSetProperty(vtctx->session,
1147  kVTCompressionPropertyKey_MoreFramesBeforeStart,
1148  kCFBooleanTrue);
1149 
1150  if (status == kVTPropertyNotSupportedErr) {
1151  av_log(avctx, AV_LOG_WARNING, "frames_before property is not supported on this device. Ignoring.\n");
1152  } else if (status) {
1153  av_log(avctx, AV_LOG_ERROR, "Error setting frames_before property: %d\n", status);
1154  }
1155  }
1156 
1157  if (vtctx->frames_after) {
1158  status = VTSessionSetProperty(vtctx->session,
1159  kVTCompressionPropertyKey_MoreFramesAfterEnd,
1160  kCFBooleanTrue);
1161 
1162  if (status == kVTPropertyNotSupportedErr) {
1163  av_log(avctx, AV_LOG_WARNING, "frames_after property is not supported on this device. Ignoring.\n");
1164  } else if (status) {
1165  av_log(avctx, AV_LOG_ERROR, "Error setting frames_after property: %d\n", status);
1166  }
1167  }
1168 
1169  if (avctx->sample_aspect_ratio.num != 0) {
1170  CFNumberRef num;
1171  CFNumberRef den;
1172  CFMutableDictionaryRef par;
1173  AVRational *avpar = &avctx->sample_aspect_ratio;
1174 
1175  av_reduce(&avpar->num, &avpar->den,
1176  avpar->num, avpar->den,
1177  0xFFFFFFFF);
1178 
1179  num = CFNumberCreate(kCFAllocatorDefault,
1180  kCFNumberIntType,
1181  &avpar->num);
1182 
1183  den = CFNumberCreate(kCFAllocatorDefault,
1184  kCFNumberIntType,
1185  &avpar->den);
1186 
1187 
1188 
1189  par = CFDictionaryCreateMutable(kCFAllocatorDefault,
1190  2,
1191  &kCFCopyStringDictionaryKeyCallBacks,
1192  &kCFTypeDictionaryValueCallBacks);
1193 
1194  if (!par || !num || !den) {
1195  if (par) CFRelease(par);
1196  if (num) CFRelease(num);
1197  if (den) CFRelease(den);
1198 
1199  return AVERROR(ENOMEM);
1200  }
1201 
1202  CFDictionarySetValue(
1203  par,
1204  kCMFormatDescriptionKey_PixelAspectRatioHorizontalSpacing,
1205  num);
1206 
1207  CFDictionarySetValue(
1208  par,
1209  kCMFormatDescriptionKey_PixelAspectRatioVerticalSpacing,
1210  den);
1211 
1212  status = VTSessionSetProperty(vtctx->session,
1213  kVTCompressionPropertyKey_PixelAspectRatio,
1214  par);
1215 
1216  CFRelease(par);
1217  CFRelease(num);
1218  CFRelease(den);
1219 
1220  if (status) {
1221  av_log(avctx,
1222  AV_LOG_ERROR,
1223  "Error setting pixel aspect ratio to %d:%d: %d.\n",
1224  avctx->sample_aspect_ratio.num,
1225  avctx->sample_aspect_ratio.den,
1226  status);
1227 
1228  return AVERROR_EXTERNAL;
1229  }
1230  }
1231 
1232 
1233  if (vtctx->transfer_function) {
1234  status = VTSessionSetProperty(vtctx->session,
1235  kVTCompressionPropertyKey_TransferFunction,
1236  vtctx->transfer_function);
1237 
1238  if (status) {
1239  av_log(avctx, AV_LOG_WARNING, "Could not set transfer function: %d\n", status);
1240  }
1241  }
1242 
1243 
1244  if (vtctx->ycbcr_matrix) {
1245  status = VTSessionSetProperty(vtctx->session,
1246  kVTCompressionPropertyKey_YCbCrMatrix,
1247  vtctx->ycbcr_matrix);
1248 
1249  if (status) {
1250  av_log(avctx, AV_LOG_WARNING, "Could not set ycbcr matrix: %d\n", status);
1251  }
1252  }
1253 
1254 
1255  if (vtctx->color_primaries) {
1256  status = VTSessionSetProperty(vtctx->session,
1257  kVTCompressionPropertyKey_ColorPrimaries,
1258  vtctx->color_primaries);
1259 
1260  if (status) {
1261  av_log(avctx, AV_LOG_WARNING, "Could not set color primaries: %d\n", status);
1262  }
1263  }
1264 
1265  if (gamma_level) {
1266  status = VTSessionSetProperty(vtctx->session,
1267  kCVImageBufferGammaLevelKey,
1268  gamma_level);
1269 
1270  if (status) {
1271  av_log(avctx, AV_LOG_WARNING, "Could not set gamma level: %d\n", status);
1272  }
1273  }
1274 
1275  if (!vtctx->has_b_frames) {
1276  status = VTSessionSetProperty(vtctx->session,
1277  kVTCompressionPropertyKey_AllowFrameReordering,
1278  kCFBooleanFalse);
1279 
1280  if (status) {
1281  av_log(avctx, AV_LOG_ERROR, "Error setting 'allow frame reordering' property: %d\n", status);
1282  return AVERROR_EXTERNAL;
1283  }
1284  }
1285 
1286  if (vtctx->entropy != VT_ENTROPY_NOT_SET) {
1287  CFStringRef entropy = vtctx->entropy == VT_CABAC ?
1288  compat_keys.kVTH264EntropyMode_CABAC:
1289  compat_keys.kVTH264EntropyMode_CAVLC;
1290 
1291  status = VTSessionSetProperty(vtctx->session,
1292  compat_keys.kVTCompressionPropertyKey_H264EntropyMode,
1293  entropy);
1294 
1295  if (status) {
1296  av_log(avctx, AV_LOG_ERROR, "Error setting entropy property: %d\n", status);
1297  }
1298  }
1299 
1300  if (vtctx->realtime) {
1301  status = VTSessionSetProperty(vtctx->session,
1302  compat_keys.kVTCompressionPropertyKey_RealTime,
1303  kCFBooleanTrue);
1304 
1305  if (status) {
1306  av_log(avctx, AV_LOG_ERROR, "Error setting realtime property: %d\n", status);
1307  }
1308  }
1309 
1310  status = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
1311  if (status) {
1312  av_log(avctx, AV_LOG_ERROR, "Error: cannot prepare encoder: %d\n", status);
1313  return AVERROR_EXTERNAL;
1314  }
1315 
1316  return 0;
1317 }
1318 
1320 {
1321  CFMutableDictionaryRef enc_info;
1322  CFMutableDictionaryRef pixel_buffer_info;
1323  CMVideoCodecType codec_type;
1324  VTEncContext *vtctx = avctx->priv_data;
1325  CFStringRef profile_level;
1326  CFNumberRef gamma_level = NULL;
1327  int status;
1328 
1329  codec_type = get_cm_codec_type(avctx->codec_id);
1330  if (!codec_type) {
1331  av_log(avctx, AV_LOG_ERROR, "Error: no mapping for AVCodecID %d\n", avctx->codec_id);
1332  return AVERROR(EINVAL);
1333  }
1334 
1335  vtctx->codec_id = avctx->codec_id;
1336 
1337  if (vtctx->codec_id == AV_CODEC_ID_H264) {
1338  vtctx->get_param_set_func = CMVideoFormatDescriptionGetH264ParameterSetAtIndex;
1339 
1340  vtctx->has_b_frames = avctx->max_b_frames > 0;
1341  if(vtctx->has_b_frames && vtctx->profile == H264_PROF_BASELINE){
1342  av_log(avctx, AV_LOG_WARNING, "Cannot use B-frames with baseline profile. Output will not contain B-frames.\n");
1343  vtctx->has_b_frames = false;
1344  }
1345 
1346  if (vtctx->entropy == VT_CABAC && vtctx->profile == H264_PROF_BASELINE) {
1347  av_log(avctx, AV_LOG_WARNING, "CABAC entropy requires 'main' or 'high' profile, but baseline was requested. Encode will not use CABAC entropy.\n");
1348  vtctx->entropy = VT_ENTROPY_NOT_SET;
1349  }
1350 
1351  if (!get_vt_h264_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1352  } else {
1353  vtctx->get_param_set_func = compat_keys.CMVideoFormatDescriptionGetHEVCParameterSetAtIndex;
1354  if (!vtctx->get_param_set_func) return AVERROR(EINVAL);
1355  if (!get_vt_hevc_profile_level(avctx, &profile_level)) return AVERROR(EINVAL);
1356  }
1357 
1358  enc_info = CFDictionaryCreateMutable(
1359  kCFAllocatorDefault,
1360  20,
1361  &kCFCopyStringDictionaryKeyCallBacks,
1362  &kCFTypeDictionaryValueCallBacks
1363  );
1364 
1365  if (!enc_info) return AVERROR(ENOMEM);
1366 
1367 #if !TARGET_OS_IPHONE
1368  if(vtctx->require_sw) {
1369  CFDictionarySetValue(enc_info,
1370  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1371  kCFBooleanFalse);
1372  } else if (!vtctx->allow_sw) {
1373  CFDictionarySetValue(enc_info,
1374  compat_keys.kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder,
1375  kCFBooleanTrue);
1376  } else {
1377  CFDictionarySetValue(enc_info,
1378  compat_keys.kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder,
1379  kCFBooleanTrue);
1380  }
1381 #endif
1382 
1383  if (avctx->pix_fmt != AV_PIX_FMT_VIDEOTOOLBOX) {
1384  status = create_cv_pixel_buffer_info(avctx, &pixel_buffer_info);
1385  if (status)
1386  goto init_cleanup;
1387  } else {
1388  pixel_buffer_info = NULL;
1389  }
1390 
1391  vtctx->dts_delta = vtctx->has_b_frames ? -1 : 0;
1392 
1393  get_cv_transfer_function(avctx, &vtctx->transfer_function, &gamma_level);
1394  get_cv_ycbcr_matrix(avctx, &vtctx->ycbcr_matrix);
1395  get_cv_color_primaries(avctx, &vtctx->color_primaries);
1396 
1397 
1398  if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
1399  status = vtenc_populate_extradata(avctx,
1400  codec_type,
1401  profile_level,
1402  gamma_level,
1403  enc_info,
1404  pixel_buffer_info);
1405  if (status)
1406  goto init_cleanup;
1407  }
1408 
1409  status = vtenc_create_encoder(avctx,
1410  codec_type,
1411  profile_level,
1412  gamma_level,
1413  enc_info,
1414  pixel_buffer_info,
1415  &vtctx->session);
1416 
1417 init_cleanup:
1418  if (gamma_level)
1419  CFRelease(gamma_level);
1420 
1421  if (pixel_buffer_info)
1422  CFRelease(pixel_buffer_info);
1423 
1424  CFRelease(enc_info);
1425 
1426  return status;
1427 }
1428 
1430 {
1431  VTEncContext *vtctx = avctx->priv_data;
1432  CFBooleanRef has_b_frames_cfbool;
1433  int status;
1434 
1435  pthread_once(&once_ctrl, loadVTEncSymbols);
1436 
1437  pthread_mutex_init(&vtctx->lock, NULL);
1439 
1440  vtctx->session = NULL;
1441  status = vtenc_configure_encoder(avctx);
1442  if (status) return status;
1443 
1444  status = VTSessionCopyProperty(vtctx->session,
1445  kVTCompressionPropertyKey_AllowFrameReordering,
1446  kCFAllocatorDefault,
1447  &has_b_frames_cfbool);
1448 
1449  if (!status && has_b_frames_cfbool) {
1450  //Some devices don't output B-frames for main profile, even if requested.
1451  vtctx->has_b_frames = CFBooleanGetValue(has_b_frames_cfbool);
1452  CFRelease(has_b_frames_cfbool);
1453  }
1454  avctx->has_b_frames = vtctx->has_b_frames;
1455 
1456  return 0;
1457 }
1458 
1459 static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
1460 {
1461  CFArrayRef attachments;
1462  CFDictionaryRef attachment;
1463  CFBooleanRef not_sync;
1464  CFIndex len;
1465 
1466  attachments = CMSampleBufferGetSampleAttachmentsArray(buffer, false);
1467  len = !attachments ? 0 : CFArrayGetCount(attachments);
1468 
1469  if (!len) {
1470  *is_key_frame = true;
1471  return;
1472  }
1473 
1474  attachment = CFArrayGetValueAtIndex(attachments, 0);
1475 
1476  if (CFDictionaryGetValueIfPresent(attachment,
1477  kCMSampleAttachmentKey_NotSync,
1478  (const void **)&not_sync))
1479  {
1480  *is_key_frame = !CFBooleanGetValue(not_sync);
1481  } else {
1482  *is_key_frame = true;
1483  }
1484 }
1485 
1486 static int is_post_sei_nal_type(int nal_type){
1487  return nal_type != H264_NAL_SEI &&
1488  nal_type != H264_NAL_SPS &&
1489  nal_type != H264_NAL_PPS &&
1490  nal_type != H264_NAL_AUD;
1491 }
1492 
1493 /*
1494  * Finds the sei message start/size of type find_sei_type.
1495  * If more than one of that type exists, the last one is returned.
1496  */
1497 static int find_sei_end(AVCodecContext *avctx,
1498  uint8_t *nal_data,
1499  size_t nal_size,
1500  uint8_t **sei_end)
1501 {
1502  int nal_type;
1503  size_t sei_payload_size = 0;
1504  int sei_payload_type = 0;
1505  *sei_end = NULL;
1506  uint8_t *nal_start = nal_data;
1507 
1508  if (!nal_size)
1509  return 0;
1510 
1511  nal_type = *nal_data & 0x1F;
1512  if (nal_type != H264_NAL_SEI)
1513  return 0;
1514 
1515  nal_data++;
1516  nal_size--;
1517 
1518  if (nal_data[nal_size - 1] == 0x80)
1519  nal_size--;
1520 
1521  while (nal_size > 0 && *nal_data > 0) {
1522  do{
1523  sei_payload_type += *nal_data;
1524  nal_data++;
1525  nal_size--;
1526  } while (nal_size > 0 && *nal_data == 0xFF);
1527 
1528  if (!nal_size) {
1529  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing type.\n");
1530  return AVERROR_INVALIDDATA;
1531  }
1532 
1533  do{
1534  sei_payload_size += *nal_data;
1535  nal_data++;
1536  nal_size--;
1537  } while (nal_size > 0 && *nal_data == 0xFF);
1538 
1539  if (nal_size < sei_payload_size) {
1540  av_log(avctx, AV_LOG_ERROR, "Unexpected end of SEI NAL Unit parsing size.\n");
1541  return AVERROR_INVALIDDATA;
1542  }
1543 
1544  nal_data += sei_payload_size;
1545  nal_size -= sei_payload_size;
1546  }
1547 
1548  *sei_end = nal_data;
1549 
1550  return nal_data - nal_start + 1;
1551 }
1552 
1553 /**
1554  * Copies the data inserting emulation prevention bytes as needed.
1555  * Existing data in the destination can be taken into account by providing
1556  * dst with a dst_offset > 0.
1557  *
1558  * @return The number of bytes copied on success. On failure, the negative of
1559  * the number of bytes needed to copy src is returned.
1560  */
1561 static int copy_emulation_prev(const uint8_t *src,
1562  size_t src_size,
1563  uint8_t *dst,
1564  ssize_t dst_offset,
1565  size_t dst_size)
1566 {
1567  int zeros = 0;
1568  int wrote_bytes;
1569  uint8_t* dst_start;
1570  uint8_t* dst_end = dst + dst_size;
1571  const uint8_t* src_end = src + src_size;
1572  int start_at = dst_offset > 2 ? dst_offset - 2 : 0;
1573  int i;
1574  for (i = start_at; i < dst_offset && i < dst_size; i++) {
1575  if (!dst[i])
1576  zeros++;
1577  else
1578  zeros = 0;
1579  }
1580 
1581  dst += dst_offset;
1582  dst_start = dst;
1583  for (; src < src_end; src++, dst++) {
1584  if (zeros == 2) {
1585  int insert_ep3_byte = *src <= 3;
1586  if (insert_ep3_byte) {
1587  if (dst < dst_end)
1588  *dst = 3;
1589  dst++;
1590  }
1591 
1592  zeros = 0;
1593  }
1594 
1595  if (dst < dst_end)
1596  *dst = *src;
1597 
1598  if (!*src)
1599  zeros++;
1600  else
1601  zeros = 0;
1602  }
1603 
1604  wrote_bytes = dst - dst_start;
1605 
1606  if (dst > dst_end)
1607  return -wrote_bytes;
1608 
1609  return wrote_bytes;
1610 }
1611 
1612 static int write_sei(const ExtraSEI *sei,
1613  int sei_type,
1614  uint8_t *dst,
1615  size_t dst_size)
1616 {
1617  uint8_t *sei_start = dst;
1618  size_t remaining_sei_size = sei->size;
1619  size_t remaining_dst_size = dst_size;
1620  int header_bytes;
1621  int bytes_written;
1622  ssize_t offset;
1623 
1624  if (!remaining_dst_size)
1625  return AVERROR_BUFFER_TOO_SMALL;
1626 
1627  while (sei_type && remaining_dst_size != 0) {
1628  int sei_byte = sei_type > 255 ? 255 : sei_type;
1629  *dst = sei_byte;
1630 
1631  sei_type -= sei_byte;
1632  dst++;
1633  remaining_dst_size--;
1634  }
1635 
1636  if (!dst_size)
1637  return AVERROR_BUFFER_TOO_SMALL;
1638 
1639  while (remaining_sei_size && remaining_dst_size != 0) {
1640  int size_byte = remaining_sei_size > 255 ? 255 : remaining_sei_size;
1641  *dst = size_byte;
1642 
1643  remaining_sei_size -= size_byte;
1644  dst++;
1645  remaining_dst_size--;
1646  }
1647 
1648  if (remaining_dst_size < sei->size)
1649  return AVERROR_BUFFER_TOO_SMALL;
1650 
1651  header_bytes = dst - sei_start;
1652 
1653  offset = header_bytes;
1654  bytes_written = copy_emulation_prev(sei->data,
1655  sei->size,
1656  sei_start,
1657  offset,
1658  dst_size);
1659  if (bytes_written < 0)
1660  return AVERROR_BUFFER_TOO_SMALL;
1661 
1662  bytes_written += header_bytes;
1663  return bytes_written;
1664 }
1665 
1666 /**
1667  * Copies NAL units and replaces length codes with
1668  * H.264 Annex B start codes. On failure, the contents of
1669  * dst_data may have been modified.
1670  *
1671  * @param length_code_size Byte length of each length code
1672  * @param sample_buffer NAL units prefixed with length codes.
1673  * @param sei Optional A53 closed captions SEI data.
1674  * @param dst_data Must be zeroed before calling this function.
1675  * Contains the copied NAL units prefixed with
1676  * start codes when the function returns
1677  * successfully.
1678  * @param dst_size Length of dst_data
1679  * @return 0 on success
1680  * AVERROR_INVALIDDATA if length_code_size is invalid
1681  * AVERROR_BUFFER_TOO_SMALL if dst_data is too small
1682  * or if a length_code in src_data specifies data beyond
1683  * the end of its buffer.
1684  */
1686  AVCodecContext *avctx,
1687  size_t length_code_size,
1688  CMSampleBufferRef sample_buffer,
1689  ExtraSEI *sei,
1690  uint8_t *dst_data,
1691  size_t dst_size)
1692 {
1693  size_t src_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1694  size_t remaining_src_size = src_size;
1695  size_t remaining_dst_size = dst_size;
1696  size_t src_offset = 0;
1697  int wrote_sei = 0;
1698  int status;
1699  uint8_t size_buf[4];
1700  uint8_t nal_type;
1701  CMBlockBufferRef block = CMSampleBufferGetDataBuffer(sample_buffer);
1702 
1703  if (length_code_size > 4) {
1704  return AVERROR_INVALIDDATA;
1705  }
1706 
1707  while (remaining_src_size > 0) {
1708  size_t curr_src_len;
1709  size_t curr_dst_len;
1710  size_t box_len = 0;
1711  size_t i;
1712 
1713  uint8_t *dst_box;
1714 
1715  status = CMBlockBufferCopyDataBytes(block,
1716  src_offset,
1717  length_code_size,
1718  size_buf);
1719  if (status) {
1720  av_log(avctx, AV_LOG_ERROR, "Cannot copy length: %d\n", status);
1721  return AVERROR_EXTERNAL;
1722  }
1723 
1724  status = CMBlockBufferCopyDataBytes(block,
1725  src_offset + length_code_size,
1726  1,
1727  &nal_type);
1728 
1729  if (status) {
1730  av_log(avctx, AV_LOG_ERROR, "Cannot copy type: %d\n", status);
1731  return AVERROR_EXTERNAL;
1732  }
1733 
1734  nal_type &= 0x1F;
1735 
1736  for (i = 0; i < length_code_size; i++) {
1737  box_len <<= 8;
1738  box_len |= size_buf[i];
1739  }
1740 
1741  if (sei && !wrote_sei && is_post_sei_nal_type(nal_type)) {
1742  //No SEI NAL unit - insert.
1743  int wrote_bytes;
1744 
1745  memcpy(dst_data, start_code, sizeof(start_code));
1746  dst_data += sizeof(start_code);
1747  remaining_dst_size -= sizeof(start_code);
1748 
1749  *dst_data = H264_NAL_SEI;
1750  dst_data++;
1751  remaining_dst_size--;
1752 
1753  wrote_bytes = write_sei(sei,
1755  dst_data,
1756  remaining_dst_size);
1757 
1758  if (wrote_bytes < 0)
1759  return wrote_bytes;
1760 
1761  remaining_dst_size -= wrote_bytes;
1762  dst_data += wrote_bytes;
1763 
1764  if (remaining_dst_size <= 0)
1765  return AVERROR_BUFFER_TOO_SMALL;
1766 
1767  *dst_data = 0x80;
1768 
1769  dst_data++;
1770  remaining_dst_size--;
1771 
1772  wrote_sei = 1;
1773  }
1774 
1775  curr_src_len = box_len + length_code_size;
1776  curr_dst_len = box_len + sizeof(start_code);
1777 
1778  if (remaining_src_size < curr_src_len) {
1779  return AVERROR_BUFFER_TOO_SMALL;
1780  }
1781 
1782  if (remaining_dst_size < curr_dst_len) {
1783  return AVERROR_BUFFER_TOO_SMALL;
1784  }
1785 
1786  dst_box = dst_data + sizeof(start_code);
1787 
1788  memcpy(dst_data, start_code, sizeof(start_code));
1789  status = CMBlockBufferCopyDataBytes(block,
1790  src_offset + length_code_size,
1791  box_len,
1792  dst_box);
1793 
1794  if (status) {
1795  av_log(avctx, AV_LOG_ERROR, "Cannot copy data: %d\n", status);
1796  return AVERROR_EXTERNAL;
1797  }
1798 
1799  if (sei && !wrote_sei && nal_type == H264_NAL_SEI) {
1800  //Found SEI NAL unit - append.
1801  int wrote_bytes;
1802  int old_sei_length;
1803  int extra_bytes;
1804  uint8_t *new_sei;
1805  old_sei_length = find_sei_end(avctx, dst_box, box_len, &new_sei);
1806  if (old_sei_length < 0)
1807  return status;
1808 
1809  wrote_bytes = write_sei(sei,
1811  new_sei,
1812  remaining_dst_size - old_sei_length);
1813  if (wrote_bytes < 0)
1814  return wrote_bytes;
1815 
1816  if (new_sei + wrote_bytes >= dst_data + remaining_dst_size)
1817  return AVERROR_BUFFER_TOO_SMALL;
1818 
1819  new_sei[wrote_bytes++] = 0x80;
1820  extra_bytes = wrote_bytes - (dst_box + box_len - new_sei);
1821 
1822  dst_data += extra_bytes;
1823  remaining_dst_size -= extra_bytes;
1824 
1825  wrote_sei = 1;
1826  }
1827 
1828  src_offset += curr_src_len;
1829  dst_data += curr_dst_len;
1830 
1831  remaining_src_size -= curr_src_len;
1832  remaining_dst_size -= curr_dst_len;
1833  }
1834 
1835  return 0;
1836 }
1837 
1838 /**
1839  * Returns a sufficient number of bytes to contain the sei data.
1840  * It may be greater than the minimum required.
1841  */
1842 static int get_sei_msg_bytes(const ExtraSEI* sei, int type){
1843  int copied_size;
1844  if (sei->size == 0)
1845  return 0;
1846 
1847  copied_size = -copy_emulation_prev(sei->data,
1848  sei->size,
1849  NULL,
1850  0,
1851  0);
1852 
1853  if ((sei->size % 255) == 0) //may result in an extra byte
1854  copied_size++;
1855 
1856  return copied_size + sei->size / 255 + 1 + type / 255 + 1;
1857 }
1858 
1860  AVCodecContext *avctx,
1861  CMSampleBufferRef sample_buffer,
1862  AVPacket *pkt,
1863  ExtraSEI *sei)
1864 {
1865  VTEncContext *vtctx = avctx->priv_data;
1866 
1867  int status;
1868  bool is_key_frame;
1869  bool add_header;
1870  size_t length_code_size;
1871  size_t header_size = 0;
1872  size_t in_buf_size;
1873  size_t out_buf_size;
1874  size_t sei_nalu_size = 0;
1875  int64_t dts_delta;
1876  int64_t time_base_num;
1877  int nalu_count;
1878  CMTime pts;
1879  CMTime dts;
1880  CMVideoFormatDescriptionRef vid_fmt;
1881 
1882 
1883  vtenc_get_frame_info(sample_buffer, &is_key_frame);
1884  status = get_length_code_size(avctx, sample_buffer, &length_code_size);
1885  if (status) return status;
1886 
1887  add_header = is_key_frame && !(avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER);
1888 
1889  if (add_header) {
1890  vid_fmt = CMSampleBufferGetFormatDescription(sample_buffer);
1891  if (!vid_fmt) {
1892  av_log(avctx, AV_LOG_ERROR, "Cannot get format description.\n");
1893  return AVERROR_EXTERNAL;
1894  }
1895 
1896  int status = get_params_size(avctx, vid_fmt, &header_size);
1897  if (status) return status;
1898  }
1899 
1900  status = count_nalus(length_code_size, sample_buffer, &nalu_count);
1901  if(status)
1902  return status;
1903 
1904  if (sei) {
1905  size_t msg_size = get_sei_msg_bytes(sei,
1907 
1908  sei_nalu_size = sizeof(start_code) + 1 + msg_size + 1;
1909  }
1910 
1911  in_buf_size = CMSampleBufferGetTotalSampleSize(sample_buffer);
1912  out_buf_size = header_size +
1913  in_buf_size +
1914  sei_nalu_size +
1915  nalu_count * ((int)sizeof(start_code) - (int)length_code_size);
1916 
1917  status = ff_alloc_packet2(avctx, pkt, out_buf_size, out_buf_size);
1918  if (status < 0)
1919  return status;
1920 
1921  if (add_header) {
1922  status = copy_param_sets(avctx, vid_fmt, pkt->data, out_buf_size);
1923  if(status) return status;
1924  }
1925 
1926  status = copy_replace_length_codes(
1927  avctx,
1928  length_code_size,
1929  sample_buffer,
1930  sei,
1931  pkt->data + header_size,
1932  pkt->size - header_size
1933  );
1934 
1935  if (status) {
1936  av_log(avctx, AV_LOG_ERROR, "Error copying packet data: %d\n", status);
1937  return status;
1938  }
1939 
1940  if (is_key_frame) {
1941  pkt->flags |= AV_PKT_FLAG_KEY;
1942  }
1943 
1944  pts = CMSampleBufferGetPresentationTimeStamp(sample_buffer);
1945  dts = CMSampleBufferGetDecodeTimeStamp (sample_buffer);
1946 
1947  if (CMTIME_IS_INVALID(dts)) {
1948  if (!vtctx->has_b_frames) {
1949  dts = pts;
1950  } else {
1951  av_log(avctx, AV_LOG_ERROR, "DTS is invalid.\n");
1952  return AVERROR_EXTERNAL;
1953  }
1954  }
1955 
1956  dts_delta = vtctx->dts_delta >= 0 ? vtctx->dts_delta : 0;
1957  time_base_num = avctx->time_base.num;
1958  pkt->pts = pts.value / time_base_num;
1959  pkt->dts = dts.value / time_base_num - dts_delta;
1960  pkt->size = out_buf_size;
1961 
1962  return 0;
1963 }
1964 
1965 /*
1966  * contiguous_buf_size is 0 if not contiguous, and the size of the buffer
1967  * containing all planes if so.
1968  */
1970  AVCodecContext *avctx,
1971  const AVFrame *frame,
1972  int *color,
1973  int *plane_count,
1974  size_t *widths,
1975  size_t *heights,
1976  size_t *strides,
1977  size_t *contiguous_buf_size)
1978 {
1979  VTEncContext *vtctx = avctx->priv_data;
1980  int av_format = frame->format;
1981  int av_color_range = frame->color_range;
1982  int i;
1983  int range_guessed;
1984  int status;
1985 
1986  status = get_cv_pixel_format(avctx, av_format, av_color_range, color, &range_guessed);
1987  if (status) {
1988  av_log(avctx,
1989  AV_LOG_ERROR,
1990  "Could not get pixel format for color format '%s' range '%s'.\n",
1991  av_get_pix_fmt_name(av_format),
1992  av_color_range > AVCOL_RANGE_UNSPECIFIED &&
1993  av_color_range < AVCOL_RANGE_NB ?
1994  av_color_range_name(av_color_range) :
1995  "Unknown");
1996 
1997  return AVERROR(EINVAL);
1998  }
1999 
2000  if (range_guessed) {
2001  if (!vtctx->warned_color_range) {
2002  vtctx->warned_color_range = true;
2003  av_log(avctx,
2005  "Color range not set for %s. Using MPEG range.\n",
2006  av_get_pix_fmt_name(av_format));
2007  }
2008  }
2009 
2010  switch (av_format) {
2011  case AV_PIX_FMT_NV12:
2012  *plane_count = 2;
2013 
2014  widths [0] = avctx->width;
2015  heights[0] = avctx->height;
2016  strides[0] = frame ? frame->linesize[0] : avctx->width;
2017 
2018  widths [1] = (avctx->width + 1) / 2;
2019  heights[1] = (avctx->height + 1) / 2;
2020  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) & -2;
2021  break;
2022 
2023  case AV_PIX_FMT_YUV420P:
2024  *plane_count = 3;
2025 
2026  widths [0] = avctx->width;
2027  heights[0] = avctx->height;
2028  strides[0] = frame ? frame->linesize[0] : avctx->width;
2029 
2030  widths [1] = (avctx->width + 1) / 2;
2031  heights[1] = (avctx->height + 1) / 2;
2032  strides[1] = frame ? frame->linesize[1] : (avctx->width + 1) / 2;
2033 
2034  widths [2] = (avctx->width + 1) / 2;
2035  heights[2] = (avctx->height + 1) / 2;
2036  strides[2] = frame ? frame->linesize[2] : (avctx->width + 1) / 2;
2037  break;
2038 
2039  case AV_PIX_FMT_P010LE:
2040  *plane_count = 2;
2041  widths[0] = avctx->width;
2042  heights[0] = avctx->height;
2043  strides[0] = frame ? frame->linesize[0] : (avctx->width * 2 + 63) & -64;
2044 
2045  widths[1] = (avctx->width + 1) / 2;
2046  heights[1] = (avctx->height + 1) / 2;
2047  strides[1] = frame ? frame->linesize[1] : ((avctx->width + 1) / 2 + 63) & -64;
2048  break;
2049 
2050  default:
2051  av_log(
2052  avctx,
2053  AV_LOG_ERROR,
2054  "Could not get frame format info for color %d range %d.\n",
2055  av_format,
2056  av_color_range);
2057 
2058  return AVERROR(EINVAL);
2059  }
2060 
2061  *contiguous_buf_size = 0;
2062  for (i = 0; i < *plane_count; i++) {
2063  if (i < *plane_count - 1 &&
2064  frame->data[i] + strides[i] * heights[i] != frame->data[i + 1]) {
2065  *contiguous_buf_size = 0;
2066  break;
2067  }
2068 
2069  *contiguous_buf_size += strides[i] * heights[i];
2070  }
2071 
2072  return 0;
2073 }
2074 
2075 //Not used on OSX - frame is never copied.
2077  const AVFrame *frame,
2078  CVPixelBufferRef cv_img,
2079  const size_t *plane_strides,
2080  const size_t *plane_rows)
2081 {
2082  int i, j;
2083  size_t plane_count;
2084  int status;
2085  int rows;
2086  int src_stride;
2087  int dst_stride;
2088  uint8_t *src_addr;
2089  uint8_t *dst_addr;
2090  size_t copy_bytes;
2091 
2092  status = CVPixelBufferLockBaseAddress(cv_img, 0);
2093  if (status) {
2094  av_log(
2095  avctx,
2096  AV_LOG_ERROR,
2097  "Error: Could not lock base address of CVPixelBuffer: %d.\n",
2098  status
2099  );
2100  }
2101 
2102  if (CVPixelBufferIsPlanar(cv_img)) {
2103  plane_count = CVPixelBufferGetPlaneCount(cv_img);
2104  for (i = 0; frame->data[i]; i++) {
2105  if (i == plane_count) {
2106  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2107  av_log(avctx,
2108  AV_LOG_ERROR,
2109  "Error: different number of planes in AVFrame and CVPixelBuffer.\n"
2110  );
2111 
2112  return AVERROR_EXTERNAL;
2113  }
2114 
2115  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddressOfPlane(cv_img, i);
2116  src_addr = (uint8_t*)frame->data[i];
2117  dst_stride = CVPixelBufferGetBytesPerRowOfPlane(cv_img, i);
2118  src_stride = plane_strides[i];
2119  rows = plane_rows[i];
2120 
2121  if (dst_stride == src_stride) {
2122  memcpy(dst_addr, src_addr, src_stride * rows);
2123  } else {
2124  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2125 
2126  for (j = 0; j < rows; j++) {
2127  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2128  }
2129  }
2130  }
2131  } else {
2132  if (frame->data[1]) {
2133  CVPixelBufferUnlockBaseAddress(cv_img, 0);
2134  av_log(avctx,
2135  AV_LOG_ERROR,
2136  "Error: different number of planes in AVFrame and non-planar CVPixelBuffer.\n"
2137  );
2138 
2139  return AVERROR_EXTERNAL;
2140  }
2141 
2142  dst_addr = (uint8_t*)CVPixelBufferGetBaseAddress(cv_img);
2143  src_addr = (uint8_t*)frame->data[0];
2144  dst_stride = CVPixelBufferGetBytesPerRow(cv_img);
2145  src_stride = plane_strides[0];
2146  rows = plane_rows[0];
2147 
2148  if (dst_stride == src_stride) {
2149  memcpy(dst_addr, src_addr, src_stride * rows);
2150  } else {
2151  copy_bytes = dst_stride < src_stride ? dst_stride : src_stride;
2152 
2153  for (j = 0; j < rows; j++) {
2154  memcpy(dst_addr + j * dst_stride, src_addr + j * src_stride, copy_bytes);
2155  }
2156  }
2157  }
2158 
2159  status = CVPixelBufferUnlockBaseAddress(cv_img, 0);
2160  if (status) {
2161  av_log(avctx, AV_LOG_ERROR, "Error: Could not unlock CVPixelBuffer base address: %d.\n", status);
2162  return AVERROR_EXTERNAL;
2163  }
2164 
2165  return 0;
2166 }
2167 
2169  const AVFrame *frame,
2170  CVPixelBufferRef *cv_img)
2171 {
2172  int plane_count;
2173  int color;
2174  size_t widths [AV_NUM_DATA_POINTERS];
2175  size_t heights[AV_NUM_DATA_POINTERS];
2176  size_t strides[AV_NUM_DATA_POINTERS];
2177  int status;
2178  size_t contiguous_buf_size;
2179  CVPixelBufferPoolRef pix_buf_pool;
2180  VTEncContext* vtctx = avctx->priv_data;
2181 
2182  if (avctx->pix_fmt == AV_PIX_FMT_VIDEOTOOLBOX) {
2184 
2185  *cv_img = (CVPixelBufferRef)frame->data[3];
2186  av_assert0(*cv_img);
2187 
2188  CFRetain(*cv_img);
2189  return 0;
2190  }
2191 
2192  memset(widths, 0, sizeof(widths));
2193  memset(heights, 0, sizeof(heights));
2194  memset(strides, 0, sizeof(strides));
2195 
2196  status = get_cv_pixel_info(
2197  avctx,
2198  frame,
2199  &color,
2200  &plane_count,
2201  widths,
2202  heights,
2203  strides,
2204  &contiguous_buf_size
2205  );
2206 
2207  if (status) {
2208  av_log(
2209  avctx,
2210  AV_LOG_ERROR,
2211  "Error: Cannot convert format %d color_range %d: %d\n",
2212  frame->format,
2213  frame->color_range,
2214  status
2215  );
2216 
2217  return AVERROR_EXTERNAL;
2218  }
2219 
2220  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2221  if (!pix_buf_pool) {
2222  /* On iOS, the VT session is invalidated when the APP switches from
2223  * foreground to background and vice versa. Fetch the actual error code
2224  * of the VT session to detect that case and restart the VT session
2225  * accordingly. */
2226  OSStatus vtstatus;
2227 
2228  vtstatus = VTCompressionSessionPrepareToEncodeFrames(vtctx->session);
2229  if (vtstatus == kVTInvalidSessionErr) {
2230  CFRelease(vtctx->session);
2231  vtctx->session = NULL;
2232  status = vtenc_configure_encoder(avctx);
2233  if (status == 0)
2234  pix_buf_pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2235  }
2236  if (!pix_buf_pool) {
2237  av_log(avctx, AV_LOG_ERROR, "Could not get pixel buffer pool.\n");
2238  return AVERROR_EXTERNAL;
2239  }
2240  else
2241  av_log(avctx, AV_LOG_WARNING, "VT session restarted because of a "
2242  "kVTInvalidSessionErr error.\n");
2243  }
2244 
2245  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2246  pix_buf_pool,
2247  cv_img);
2248 
2249 
2250  if (status) {
2251  av_log(avctx, AV_LOG_ERROR, "Could not create pixel buffer from pool: %d.\n", status);
2252  return AVERROR_EXTERNAL;
2253  }
2254 
2255  status = copy_avframe_to_pixel_buffer(avctx, frame, *cv_img, strides, heights);
2256  if (status) {
2257  CFRelease(*cv_img);
2258  *cv_img = NULL;
2259  return status;
2260  }
2261 
2262  return 0;
2263 }
2264 
2266  CFDictionaryRef* dict_out)
2267 {
2268  CFDictionaryRef dict = NULL;
2269  if (frame->pict_type == AV_PICTURE_TYPE_I) {
2270  const void *keys[] = { kVTEncodeFrameOptionKey_ForceKeyFrame };
2271  const void *vals[] = { kCFBooleanTrue };
2272 
2273  dict = CFDictionaryCreate(NULL, keys, vals, 1, NULL, NULL);
2274  if(!dict) return AVERROR(ENOMEM);
2275  }
2276 
2277  *dict_out = dict;
2278  return 0;
2279 }
2280 
2282  VTEncContext *vtctx,
2283  const AVFrame *frame)
2284 {
2285  CMTime time;
2286  CFDictionaryRef frame_dict;
2287  CVPixelBufferRef cv_img = NULL;
2288  AVFrameSideData *side_data = NULL;
2289  ExtraSEI *sei = NULL;
2290  int status = create_cv_pixel_buffer(avctx, frame, &cv_img);
2291 
2292  if (status) return status;
2293 
2294  status = create_encoder_dict_h264(frame, &frame_dict);
2295  if (status) {
2296  CFRelease(cv_img);
2297  return status;
2298  }
2299 
2300  side_data = av_frame_get_side_data(frame, AV_FRAME_DATA_A53_CC);
2301  if (vtctx->a53_cc && side_data && side_data->size) {
2302  sei = av_mallocz(sizeof(*sei));
2303  if (!sei) {
2304  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2305  } else {
2306  int ret = ff_alloc_a53_sei(frame, 0, &sei->data, &sei->size);
2307  if (ret < 0) {
2308  av_log(avctx, AV_LOG_ERROR, "Not enough memory for closed captions, skipping\n");
2309  av_free(sei);
2310  sei = NULL;
2311  }
2312  }
2313  }
2314 
2315  time = CMTimeMake(frame->pts * avctx->time_base.num, avctx->time_base.den);
2316  status = VTCompressionSessionEncodeFrame(
2317  vtctx->session,
2318  cv_img,
2319  time,
2320  kCMTimeInvalid,
2321  frame_dict,
2322  sei,
2323  NULL
2324  );
2325 
2326  if (frame_dict) CFRelease(frame_dict);
2327  CFRelease(cv_img);
2328 
2329  if (status) {
2330  av_log(avctx, AV_LOG_ERROR, "Error: cannot encode frame: %d\n", status);
2331  return AVERROR_EXTERNAL;
2332  }
2333 
2334  return 0;
2335 }
2336 
2338  AVCodecContext *avctx,
2339  AVPacket *pkt,
2340  const AVFrame *frame,
2341  int *got_packet)
2342 {
2343  VTEncContext *vtctx = avctx->priv_data;
2344  bool get_frame;
2345  int status;
2346  CMSampleBufferRef buf = NULL;
2347  ExtraSEI *sei = NULL;
2348 
2349  if (frame) {
2350  status = vtenc_send_frame(avctx, vtctx, frame);
2351 
2352  if (status) {
2353  status = AVERROR_EXTERNAL;
2354  goto end_nopkt;
2355  }
2356 
2357  if (vtctx->frame_ct_in == 0) {
2358  vtctx->first_pts = frame->pts;
2359  } else if(vtctx->frame_ct_in == 1 && vtctx->has_b_frames) {
2360  vtctx->dts_delta = frame->pts - vtctx->first_pts;
2361  }
2362 
2363  vtctx->frame_ct_in++;
2364  } else if(!vtctx->flushing) {
2365  vtctx->flushing = true;
2366 
2367  status = VTCompressionSessionCompleteFrames(vtctx->session,
2368  kCMTimeIndefinite);
2369 
2370  if (status) {
2371  av_log(avctx, AV_LOG_ERROR, "Error flushing frames: %d\n", status);
2372  status = AVERROR_EXTERNAL;
2373  goto end_nopkt;
2374  }
2375  }
2376 
2377  *got_packet = 0;
2378  get_frame = vtctx->dts_delta >= 0 || !frame;
2379  if (!get_frame) {
2380  status = 0;
2381  goto end_nopkt;
2382  }
2383 
2384  status = vtenc_q_pop(vtctx, !frame, &buf, &sei);
2385  if (status) goto end_nopkt;
2386  if (!buf) goto end_nopkt;
2387 
2388  status = vtenc_cm_to_avpacket(avctx, buf, pkt, sei);
2389  if (sei) {
2390  if (sei->data) av_free(sei->data);
2391  av_free(sei);
2392  }
2393  CFRelease(buf);
2394  if (status) goto end_nopkt;
2395 
2396  *got_packet = 1;
2397  return 0;
2398 
2399 end_nopkt:
2400  av_packet_unref(pkt);
2401  return status;
2402 }
2403 
2405  CMVideoCodecType codec_type,
2406  CFStringRef profile_level,
2407  CFNumberRef gamma_level,
2408  CFDictionaryRef enc_info,
2409  CFDictionaryRef pixel_buffer_info)
2410 {
2411  VTEncContext *vtctx = avctx->priv_data;
2412  int status;
2413  CVPixelBufferPoolRef pool = NULL;
2414  CVPixelBufferRef pix_buf = NULL;
2415  CMTime time;
2416  CMSampleBufferRef buf = NULL;
2417 
2418  status = vtenc_create_encoder(avctx,
2419  codec_type,
2420  profile_level,
2421  gamma_level,
2422  enc_info,
2423  pixel_buffer_info,
2424  &vtctx->session);
2425  if (status)
2426  goto pe_cleanup;
2427 
2428  pool = VTCompressionSessionGetPixelBufferPool(vtctx->session);
2429  if(!pool){
2430  av_log(avctx, AV_LOG_ERROR, "Error getting pixel buffer pool.\n");
2431  goto pe_cleanup;
2432  }
2433 
2434  status = CVPixelBufferPoolCreatePixelBuffer(NULL,
2435  pool,
2436  &pix_buf);
2437 
2438  if(status != kCVReturnSuccess){
2439  av_log(avctx, AV_LOG_ERROR, "Error creating frame from pool: %d\n", status);
2440  goto pe_cleanup;
2441  }
2442 
2443  time = CMTimeMake(0, avctx->time_base.den);
2444  status = VTCompressionSessionEncodeFrame(vtctx->session,
2445  pix_buf,
2446  time,
2447  kCMTimeInvalid,
2448  NULL,
2449  NULL,
2450  NULL);
2451 
2452  if (status) {
2453  av_log(avctx,
2454  AV_LOG_ERROR,
2455  "Error sending frame for extradata: %d\n",
2456  status);
2457 
2458  goto pe_cleanup;
2459  }
2460 
2461  //Populates extradata - output frames are flushed and param sets are available.
2462  status = VTCompressionSessionCompleteFrames(vtctx->session,
2463  kCMTimeIndefinite);
2464 
2465  if (status)
2466  goto pe_cleanup;
2467 
2468  status = vtenc_q_pop(vtctx, 0, &buf, NULL);
2469  if (status) {
2470  av_log(avctx, AV_LOG_ERROR, "popping: %d\n", status);
2471  goto pe_cleanup;
2472  }
2473 
2474  CFRelease(buf);
2475 
2476 
2477 
2478 pe_cleanup:
2479  if(vtctx->session)
2480  CFRelease(vtctx->session);
2481 
2482  vtctx->session = NULL;
2483  vtctx->frame_ct_out = 0;
2484 
2485  av_assert0(status != 0 || (avctx->extradata && avctx->extradata_size > 0));
2486 
2487  return status;
2488 }
2489 
2491 {
2492  VTEncContext *vtctx = avctx->priv_data;
2493 
2494  if(!vtctx->session) {
2496  pthread_mutex_destroy(&vtctx->lock);
2497  return 0;
2498  }
2499 
2500  VTCompressionSessionCompleteFrames(vtctx->session,
2501  kCMTimeIndefinite);
2502  clear_frame_queue(vtctx);
2504  pthread_mutex_destroy(&vtctx->lock);
2505  CFRelease(vtctx->session);
2506  vtctx->session = NULL;
2507 
2508  if (vtctx->color_primaries) {
2509  CFRelease(vtctx->color_primaries);
2510  vtctx->color_primaries = NULL;
2511  }
2512 
2513  if (vtctx->transfer_function) {
2514  CFRelease(vtctx->transfer_function);
2515  vtctx->transfer_function = NULL;
2516  }
2517 
2518  if (vtctx->ycbcr_matrix) {
2519  CFRelease(vtctx->ycbcr_matrix);
2520  vtctx->ycbcr_matrix = NULL;
2521  }
2522 
2523  return 0;
2524 }
2525 
2526 static const enum AVPixelFormat avc_pix_fmts[] = {
2531 };
2532 
2533 static const enum AVPixelFormat hevc_pix_fmts[] = {
2539 };
2540 
2541 #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
2542 #define COMMON_OPTIONS \
2543  { "allow_sw", "Allow software encoding", OFFSET(allow_sw), AV_OPT_TYPE_BOOL, \
2544  { .i64 = 0 }, 0, 1, VE }, \
2545  { "require_sw", "Require software encoding", OFFSET(require_sw), AV_OPT_TYPE_BOOL, \
2546  { .i64 = 0 }, 0, 1, VE }, \
2547  { "realtime", "Hint that encoding should happen in real-time if not faster (e.g. capturing from camera).", \
2548  OFFSET(realtime), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2549  { "frames_before", "Other frames will come before the frames in this session. This helps smooth concatenation issues.", \
2550  OFFSET(frames_before), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, \
2551  { "frames_after", "Other frames will come after the frames in this session. This helps smooth concatenation issues.", \
2552  OFFSET(frames_after), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE },
2553 
2554 #define OFFSET(x) offsetof(VTEncContext, x)
2555 static const AVOption h264_options[] = {
2556  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = H264_PROF_AUTO }, H264_PROF_AUTO, H264_PROF_COUNT, VE, "profile" },
2557  { "baseline", "Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_BASELINE }, INT_MIN, INT_MAX, VE, "profile" },
2558  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2559  { "high", "High Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_HIGH }, INT_MIN, INT_MAX, VE, "profile" },
2560  { "extended", "Extend Profile", 0, AV_OPT_TYPE_CONST, { .i64 = H264_PROF_EXTENDED }, INT_MIN, INT_MAX, VE, "profile" },
2561 
2562  { "level", "Level", OFFSET(level), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 52, VE, "level" },
2563  { "1.3", "Level 1.3, only available with Baseline Profile", 0, AV_OPT_TYPE_CONST, { .i64 = 13 }, INT_MIN, INT_MAX, VE, "level" },
2564  { "3.0", "Level 3.0", 0, AV_OPT_TYPE_CONST, { .i64 = 30 }, INT_MIN, INT_MAX, VE, "level" },
2565  { "3.1", "Level 3.1", 0, AV_OPT_TYPE_CONST, { .i64 = 31 }, INT_MIN, INT_MAX, VE, "level" },
2566  { "3.2", "Level 3.2", 0, AV_OPT_TYPE_CONST, { .i64 = 32 }, INT_MIN, INT_MAX, VE, "level" },
2567  { "4.0", "Level 4.0", 0, AV_OPT_TYPE_CONST, { .i64 = 40 }, INT_MIN, INT_MAX, VE, "level" },
2568  { "4.1", "Level 4.1", 0, AV_OPT_TYPE_CONST, { .i64 = 41 }, INT_MIN, INT_MAX, VE, "level" },
2569  { "4.2", "Level 4.2", 0, AV_OPT_TYPE_CONST, { .i64 = 42 }, INT_MIN, INT_MAX, VE, "level" },
2570  { "5.0", "Level 5.0", 0, AV_OPT_TYPE_CONST, { .i64 = 50 }, INT_MIN, INT_MAX, VE, "level" },
2571  { "5.1", "Level 5.1", 0, AV_OPT_TYPE_CONST, { .i64 = 51 }, INT_MIN, INT_MAX, VE, "level" },
2572  { "5.2", "Level 5.2", 0, AV_OPT_TYPE_CONST, { .i64 = 52 }, INT_MIN, INT_MAX, VE, "level" },
2573 
2574  { "coder", "Entropy coding", OFFSET(entropy), AV_OPT_TYPE_INT, { .i64 = VT_ENTROPY_NOT_SET }, VT_ENTROPY_NOT_SET, VT_CABAC, VE, "coder" },
2575  { "cavlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2576  { "vlc", "CAVLC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CAVLC }, INT_MIN, INT_MAX, VE, "coder" },
2577  { "cabac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2578  { "ac", "CABAC entropy coding", 0, AV_OPT_TYPE_CONST, { .i64 = VT_CABAC }, INT_MIN, INT_MAX, VE, "coder" },
2579 
2580  { "a53cc", "Use A53 Closed Captions (if available)", OFFSET(a53_cc), AV_OPT_TYPE_BOOL, {.i64 = 1}, 0, 1, VE },
2581 
2583  { NULL },
2584 };
2585 
2587  .class_name = "h264_videotoolbox",
2588  .item_name = av_default_item_name,
2589  .option = h264_options,
2590  .version = LIBAVUTIL_VERSION_INT,
2591 };
2592 
2594  .name = "h264_videotoolbox",
2595  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.264 Encoder"),
2596  .type = AVMEDIA_TYPE_VIDEO,
2597  .id = AV_CODEC_ID_H264,
2598  .priv_data_size = sizeof(VTEncContext),
2600  .init = vtenc_init,
2601  .encode2 = vtenc_frame,
2602  .close = vtenc_close,
2603  .capabilities = AV_CODEC_CAP_DELAY,
2604  .priv_class = &h264_videotoolbox_class,
2605  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2607 };
2608 
2609 static const AVOption hevc_options[] = {
2610  { "profile", "Profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = HEVC_PROF_AUTO }, HEVC_PROF_AUTO, HEVC_PROF_COUNT, VE, "profile" },
2611  { "main", "Main Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN }, INT_MIN, INT_MAX, VE, "profile" },
2612  { "main10", "Main10 Profile", 0, AV_OPT_TYPE_CONST, { .i64 = HEVC_PROF_MAIN10 }, INT_MIN, INT_MAX, VE, "profile" },
2613 
2615  { NULL },
2616 };
2617 
2619  .class_name = "hevc_videotoolbox",
2620  .item_name = av_default_item_name,
2621  .option = hevc_options,
2622  .version = LIBAVUTIL_VERSION_INT,
2623 };
2624 
2626  .name = "hevc_videotoolbox",
2627  .long_name = NULL_IF_CONFIG_SMALL("VideoToolbox H.265 Encoder"),
2628  .type = AVMEDIA_TYPE_VIDEO,
2629  .id = AV_CODEC_ID_HEVC,
2630  .priv_data_size = sizeof(VTEncContext),
2632  .init = vtenc_init,
2633  .encode2 = vtenc_frame,
2634  .close = vtenc_close,
2635  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE,
2636  .priv_class = &hevc_videotoolbox_class,
2637  .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE |
2639  .wrapper_name = "videotoolbox",
2640 };
#define FF_CODEC_CAP_INIT_CLEANUP
The codec allows calling the close function for deallocation even if the init function returned a fai...
Definition: internal.h:48
ITU-R BT2020 for 12-bit system.
Definition: pixfmt.h:499
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:514
#define NULL
Definition: coverity.c:32
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:59
static void vtenc_q_push(VTEncContext *vtctx, CMSampleBufferRef buffer, ExtraSEI *sei)
#define AV_NUM_DATA_POINTERS
Definition: frame.h:309
static CMVideoCodecType get_cm_codec_type(enum AVCodecID id)
static av_always_inline int pthread_mutex_destroy(pthread_mutex_t *mutex)
Definition: os2threads.h:112
pthread_cond_t cv_sample_sent
This structure describes decoded (raw) audio or video data.
Definition: frame.h:308
#define pthread_mutex_lock(a)
Definition: ffprobe.c:62
static av_always_inline int pthread_cond_wait(pthread_cond_t *cond, pthread_mutex_t *mutex)
Definition: os2threads.h:192
AVOption.
Definition: opt.h:248
"Linear transfer characteristics"
Definition: pixfmt.h:492
BufNode * q_head
struct BufNode * next
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
static const AVClass hevc_videotoolbox_class
int64_t bit_rate
the average bitrate
Definition: avcodec.h:576
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
hardware decoding through Videotoolbox
Definition: pixfmt.h:282
static av_cold int init(AVCodecContext *avctx)
Definition: avrndec.c:35
static int set_extradata(AVCodecContext *avctx, CMSampleBufferRef sample_buffer)
int max_b_frames
maximum number of B-frames between non-B-frames Note: The output will be delayed by max_b_frames+1 re...
Definition: avcodec.h:786
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601 ...
Definition: pixfmt.h:518
static int copy_param_sets(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, uint8_t *dst, size_t dst_size)
#define AV_CODEC_CAP_HARDWARE
Codec is backed by a hardware implementation.
Definition: codec.h:150
static int create_cv_pixel_buffer_info(AVCodecContext *avctx, CFMutableDictionaryRef *dict)
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: avcodec.h:1161
int num
Numerator.
Definition: rational.h:59
int size
Definition: packet.h:364
static int vtenc_configure_encoder(AVCodecContext *avctx)
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:235
static int write_sei(const ExtraSEI *sei, int sei_type, uint8_t *dst, size_t dst_size)
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel...
Definition: avcodec.h:905
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:519
enum AVMediaType codec_type
Definition: rtp.c:37
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:736
GLint GLenum type
Definition: opengl_enc.c:104
CFStringRef kVTProfileLevel_H264_High_5_2
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
static AVPacket pkt
AVCodec.
Definition: codec.h:190
static av_always_inline int pthread_cond_destroy(pthread_cond_t *cond)
Definition: os2threads.h:144
AVColorTransferCharacteristic
Color Transfer Characteristic.
Definition: pixfmt.h:483
functionally identical to above
Definition: pixfmt.h:520
const char * av_color_space_name(enum AVColorSpace space)
Definition: pixdesc.c:2966
int64_t require_sw
CFStringRef kVTCompressionPropertyKey_H264EntropyMode
static int copy_replace_length_codes(AVCodecContext *avctx, size_t length_code_size, CMSampleBufferRef sample_buffer, ExtraSEI *sei, uint8_t *dst_data, size_t dst_size)
Copies NAL units and replaces length codes with H.264 Annex B start codes.
AVRational time_base
This is the fundamental unit of time (in seconds) in terms of which frame timestamps are represented...
Definition: avcodec.h:649
static int is_post_sei_nal_type(int nal_type)
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:751
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
#define AV_CODEC_CAP_DELAY
Encoder or decoder requires flushing with NULL input at the end in order to give the complete and cor...
Definition: codec.h:75
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
static int vtenc_cm_to_avpacket(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, AVPacket *pkt, ExtraSEI *sei)
int ff_alloc_packet2(AVCodecContext *avctx, AVPacket *avpkt, int64_t size, int64_t min_size)
Check AVPacket size and/or allocate data.
Definition: encode.c:33
AVCodec ff_hevc_videotoolbox_encoder
CFStringRef kCVImageBufferTransferFunction_ITU_R_2020
#define FF_CODEC_CAP_INIT_THREADSAFE
The codec does not modify any global variables in the init function, allowing to call the init functi...
Definition: internal.h:40
The exact code depends on how similar the blocks are and how related they are to the block
uint8_t
#define av_cold
Definition: attributes.h:88
#define av_malloc(s)
CFStringRef kVTProfileLevel_HEVC_Main_AutoLevel
AVOptions.
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
static const AVOption hevc_options[]
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf offset
const char * av_color_range_name(enum AVColorRange range)
Definition: pixdesc.c:2899
also ITU-R BT470M / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:488
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:401
CFStringRef kVTProfileLevel_H264_Baseline_5_2
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:627
static int get_cv_pixel_info(AVCodecContext *avctx, const AVFrame *frame, int *color, int *plane_count, size_t *widths, size_t *heights, size_t *strides, size_t *contiguous_buf_size)
Structure to hold side data for an AVFrame.
Definition: frame.h:214
uint8_t * data
Definition: packet.h:363
int64_t frame_ct_in
Not part of ABI.
Definition: pixfmt.h:587
static av_always_inline int pthread_cond_signal(pthread_cond_t *cond)
Definition: os2threads.h:152
static int get_length_code_size(AVCodecContext *avctx, CMSampleBufferRef sample_buffer, size_t *size)
CFStringRef kVTProfileLevel_H264_High_4_0
AVColorRange
Visual content value range.
Definition: pixfmt.h:551
ptrdiff_t size
Definition: opengl_enc.c:100
int av_reduce(int *dst_num, int *dst_den, int64_t num, int64_t den, int64_t max)
Reduce a fraction.
Definition: rational.c:35
int ff_alloc_a53_sei(const AVFrame *frame, size_t prefix_len, void **data, size_t *sei_size)
Check AVFrame for A53 side data and allocate and fill SEI message with A53 info.
Definition: atsc_a53.c:25
AVColorPrimaries
Chromaticity coordinates of the source primaries.
Definition: pixfmt.h:458
static av_cold int vtenc_frame(AVCodecContext *avctx, AVPacket *pkt, const AVFrame *frame, int *got_packet)
#define av_log(a,...)
#define AV_PKT_FLAG_KEY
The packet contains a keyframe.
Definition: packet.h:401
CFStringRef kVTProfileLevel_HEVC_Main10_AutoLevel
H.264 common definitions.
#define GET_SYM(symbol, defaultVal)
#define src
Definition: vp8dsp.c:254
CFStringRef kVTProfileLevel_H264_Baseline_5_1
AVCodecID
Identify the syntax and semantics of the bitstream.
Definition: codec_id.h:46
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
int has_b_frames
Size of the frame reordering buffer in the decoder.
Definition: avcodec.h:816
CFStringRef kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder
static void vtenc_get_frame_info(CMSampleBufferRef buffer, bool *is_key_frame)
CFStringRef kVTProfileLevel_H264_Baseline_5_0
CFStringRef kVTProfileLevel_H264_Main_4_2
static int find_sei_end(AVCodecContext *avctx, uint8_t *nal_data, size_t nal_size, uint8_t **sei_end)
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:153
BufNode * q_tail
enum AVColorRange color_range
MPEG vs JPEG YUV range.
Definition: frame.h:552
ATSC A53 Part 4 Closed Captions.
Definition: frame.h:58
static void add_color_attr(AVCodecContext *avctx, CFMutableDictionaryRef dict)
CFStringRef kVTProfileLevel_H264_High_4_1
static int get_params_size(AVCodecContext *avctx, CMVideoFormatDescriptionRef vid_fmt, size_t *size)
Get the parameter sets from a CMSampleBufferRef.
int flags
AV_CODEC_FLAG_*.
Definition: avcodec.h:606
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
also ITU-R BT1361 / IEC 61966-2-4 / SMPTE RP177 Annex B
Definition: pixfmt.h:460
ExtraSEI * sei
CFStringRef kVTProfileLevel_H264_Extended_5_0
simple assert() macros that are a bit more flexible than ISO C assert().
const char * name
Name of the codec implementation.
Definition: codec.h:197
CFStringRef kVTProfileLevel_H264_High_3_0
VTH264Entropy
VT_HEVCProfile
CFStringRef kVTProfileLevel_H264_Baseline_4_0
GLsizei count
Definition: opengl_enc.c:108
static int get_frame(AVFilterContext *ctx, int is_second)
Definition: vf_nnedi.c:689
static bool get_vt_h264_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
int flags
A combination of AV_PKT_FLAG values.
Definition: packet.h:369
const char * av_color_primaries_name(enum AVColorPrimaries primaries)
Definition: pixdesc.c:2918
int64_t frames_before
static int vtenc_create_encoder(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info, VTCompressionSessionRef *session)
VTCompressionSessionRef session
CFStringRef color_primaries
CFStringRef kVTProfileLevel_H264_High_4_2
MIPS optimizations info
Definition: mips.txt:2
static int get_cv_transfer_function(AVCodecContext *avctx, CFStringRef *transfer_fnc, CFNumberRef *gamma_level)
SMPTE ST 2084 for 10-, 12-, 14- and 16-bit systems.
Definition: pixfmt.h:500
enum AVPictureType pict_type
Picture type of the frame.
Definition: frame.h:391
#define AVERROR_BUFFER_TOO_SMALL
Buffer too small.
Definition: error.h:51
CMSampleBufferRef cm_buffer
int width
picture width / height.
Definition: avcodec.h:699
static void loadVTEncSymbols()
ITU-R BT2020 non-constant luminance system.
Definition: pixfmt.h:523
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM
Definition: pixfmt.h:465
static struct @168 compat_keys
getParameterSetAtIndex CMVideoFormatDescriptionGetHEVCParameterSetAtIndex
AVFormatContext * ctx
Definition: movenc.c:48
enum AVColorPrimaries color_primaries
Chromaticity coordinates of the source primaries.
Definition: avcodec.h:1140
#define OFFSET(x)
static av_always_inline int pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *attr)
Definition: os2threads.h:104
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
CFStringRef kVTH264EntropyMode_CAVLC
CFStringRef kCVImageBufferYCbCrMatrix_ITU_R_2020
enum AVCodecID codec_id
Definition: vaapi_decode.c:369
#define pthread_mutex_unlock(a)
Definition: ffprobe.c:66
like NV12, with 10bpp per component, data in the high bits, zeros in the low bits, little-endian
Definition: pixfmt.h:284
static int copy_avframe_to_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef cv_img, const size_t *plane_strides, const size_t *plane_rows)
Full range content.
Definition: pixfmt.h:586
enum AVCodecID codec_id
CFStringRef kVTProfileLevel_H264_Baseline_AutoLevel
static int vtenc_q_pop(VTEncContext *vtctx, bool wait, CMSampleBufferRef *buf, ExtraSEI **sei)
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:381
CFStringRef kVTCompressionPropertyKey_RealTime
also ITU-R BT1361
Definition: pixfmt.h:485
static int vtenc_populate_extradata(AVCodecContext *avctx, CMVideoCodecType codec_type, CFStringRef profile_level, CFNumberRef gamma_level, CFDictionaryRef enc_info, CFDictionaryRef pixel_buffer_info)
CFStringRef kCVImageBufferColorPrimaries_ITU_R_2020
static const AVClass h264_videotoolbox_class
Libavcodec external API header.
static int get_cv_pixel_format(AVCodecContext *avctx, enum AVPixelFormat fmt, enum AVColorRange range, int *av_pixel_format, int *range_guessed)
static enum AVPixelFormat hevc_pix_fmts[]
enum AVCodecID codec_id
Definition: avcodec.h:536
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:339
CFStringRef kVTProfileLevel_H264_High_3_1
main external API structure.
Definition: avcodec.h:526
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:606
static int create_encoder_dict_h264(const AVFrame *frame, CFDictionaryRef *dict_out)
static av_cold int vtenc_close(AVCodecContext *avctx)
int extradata_size
Definition: avcodec.h:628
Describe the class of an AVClass context structure.
Definition: log.h:67
CFStringRef transfer_function
static void clear_frame_queue(VTEncContext *vtctx)
static int get_cv_ycbcr_matrix(AVCodecContext *avctx, CFStringRef *matrix)
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:1154
Rational number (pair of numerator and denominator).
Definition: rational.h:58
enum AVColorTransferCharacteristic color_trc
Color Transfer Characteristic.
Definition: avcodec.h:1147
AVCodec ff_h264_videotoolbox_encoder
static int copy_emulation_prev(const uint8_t *src, size_t src_size, uint8_t *dst, ssize_t dst_offset, size_t dst_size)
Copies the data inserting emulation prevention bytes as needed.
static int vtenc_send_frame(AVCodecContext *avctx, VTEncContext *vtctx, const AVFrame *frame)
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
VT_H264Profile
static pthread_once_t once_ctrl
registered user data as specified by Rec. ITU-T T.35
Definition: h264_sei.h:33
int64_t frames_after
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:300
CFStringRef kVTProfileLevel_H264_Main_AutoLevel
mfxU16 profile
Definition: qsvenc.c:45
CFStringRef kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder
static int get_sei_msg_bytes(const ExtraSEI *sei, int type)
Returns a sufficient number of bytes to contain the sei data.
#define COMMON_OPTIONS
static int64_t pts
#define flags(name, subs,...)
Definition: cbs_av1.c:560
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:322
uint8_t level
Definition: svq3.c:205
#define AV_CODEC_FLAG_GLOBAL_HEADER
Place global headers in extradata instead of every keyframe.
Definition: avcodec.h:329
Narrow or limited range content.
Definition: pixfmt.h:569
int gop_size
the number of pictures in a group of pictures, or 0 for intra_only
Definition: avcodec.h:721
static int create_cv_pixel_buffer(AVCodecContext *avctx, const AVFrame *frame, CVPixelBufferRef *cv_img)
getParameterSetAtIndex get_param_set_func
CFStringRef kVTH264EntropyMode_CABAC
int
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
common internal api header.
_fmutex pthread_mutex_t
Definition: os2threads.h:53
CFStringRef kVTProfileLevel_H264_High_AutoLevel
const char * av_color_transfer_name(enum AVColorTransferCharacteristic transfer)
Definition: pixdesc.c:2942
OSStatus(* getParameterSetAtIndex)(CMFormatDescriptionRef videoDesc, size_t parameterSetIndex, const uint8_t **parameterSetPointerOut, size_t *parameterSetSizeOut, size_t *parameterSetCountOut, int *NALUnitHeaderLengthOut)
CFStringRef ycbcr_matrix
static void vtenc_output_callback(void *ctx, void *sourceFrameCtx, OSStatus status, VTEncodeInfoFlags flags, CMSampleBufferRef sample_buffer)
also ITU-R BT470BG
Definition: pixfmt.h:489
static void vt_release_num(CFNumberRef *refPtr)
NULL-safe release of *refPtr, and sets value to NULL.
CFStringRef kVTProfileLevel_H264_High_3_2
static av_always_inline int pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *attr)
Definition: os2threads.h:133
pthread_mutex_t lock
static int FUNC() sei(CodedBitstreamContext *ctx, RWContext *rw, H264RawSEI *current)
int den
Denominator.
Definition: rational.h:60
static bool get_vt_hevc_profile_level(AVCodecContext *avctx, CFStringRef *profile_level_val)
#define AV_INPUT_BUFFER_PADDING_SIZE
Required number of additionally allocated bytes at the end of the input bitstream for decoding...
Definition: avcodec.h:215
ARIB STD-B67, known as "Hybrid log-gamma".
Definition: pixfmt.h:504
CFStringRef kVTProfileLevel_H264_Extended_AutoLevel
void * priv_data
Definition: avcodec.h:553
#define PTHREAD_ONCE_INIT
Definition: os2threads.h:71
#define av_free(p)
int len
CFStringRef kVTProfileLevel_H264_High_5_1
static const AVOption h264_options[]
#define VE
int64_t frame_ct_out
CFStringRef kVTProfileLevel_H264_Baseline_4_2
ITU-R BT2020 for 10-bit system.
Definition: pixfmt.h:498
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: packet.h:362
also ITU-R BT601-6 525 / ITU-R BT1358 525 / ITU-R BT1700 NTSC
Definition: pixfmt.h:466
ITU-R BT2020.
Definition: pixfmt.h:469
static int count_nalus(size_t length_code_size, CMSampleBufferRef sample_buffer, int *count)
static const uint8_t start_code[]
static av_cold int vtenc_init(AVCodecContext *avctx)
const char * av_get_pix_fmt_name(enum AVPixelFormat pix_fmt)
Return the short name for a pixel format, NULL in case pix_fmt is unknown.
Definition: pixdesc.c:2489
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
static int get_cv_color_primaries(AVCodecContext *avctx, CFStringRef *primaries)
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
This structure stores compressed data.
Definition: packet.h:340
static av_always_inline int pthread_once(pthread_once_t *once_control, void(*init_routine)(void))
Definition: os2threads.h:210
static void set_async_error(VTEncContext *vtctx, int err)
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:356
int i
Definition: input.c:407
GLuint buffer
Definition: opengl_enc.c:101
CFStringRef kVTProfileLevel_H264_Main_5_1
CFStringRef kVTProfileLevel_H264_Main_5_2
int64_t rc_max_rate
maximum bitrate
Definition: avcodec.h:1404
static enum AVPixelFormat avc_pix_fmts[]