FFmpeg
mmaldec.c
Go to the documentation of this file.
1 /*
2  * MMAL Video Decoder
3  * Copyright (c) 2015 Rodger Combs
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * MMAL Video Decoder
25  */
26 
27 #include <bcm_host.h>
28 #include <interface/mmal/mmal.h>
29 #include <interface/mmal/mmal_parameters_video.h>
30 #include <interface/mmal/util/mmal_util.h>
31 #include <interface/mmal/util/mmal_util_params.h>
32 #include <interface/mmal/util/mmal_default_components.h>
33 #include <interface/mmal/vc/mmal_vc_api.h>
34 #include <stdatomic.h>
35 
36 #include "avcodec.h"
37 #include "decode.h"
38 #include "hwconfig.h"
39 #include "internal.h"
40 #include "libavutil/avassert.h"
41 #include "libavutil/buffer.h"
42 #include "libavutil/common.h"
43 #include "libavutil/imgutils.h"
44 #include "libavutil/opt.h"
45 #include "libavutil/log.h"
46 
47 typedef struct FFBufferEntry {
49  void *data;
50  size_t length;
51  int64_t pts, dts;
52  int flags;
55 
56 // MMAL_POOL_T destroys all of its MMAL_BUFFER_HEADER_Ts. If we want correct
57 // refcounting for AVFrames, we can free the MMAL_POOL_T only after all AVFrames
58 // have been unreferenced.
59 typedef struct FFPoolRef {
61  MMAL_POOL_T *pool;
62 } FFPoolRef;
63 
64 typedef struct FFBufferRef {
65  MMAL_BUFFER_HEADER_T *buffer;
67 } FFBufferRef;
68 
69 typedef struct MMALDecodeContext {
73 
74  MMAL_COMPONENT_T *decoder;
75  MMAL_QUEUE_T *queue_decoded_frames;
76  MMAL_POOL_T *pool_in;
78 
79  // Waiting input packets. Because the libavcodec API requires decoding and
80  // returning packets in lockstep, it can happen that queue_decoded_frames
81  // contains almost all surfaces - then the decoder input queue can quickly
82  // fill up and won't accept new input either. Without consuming input, the
83  // libavcodec API can't return new frames, and we have a logical deadlock.
84  // This is avoided by queuing such buffers here.
85  FFBufferEntry *waiting_buffers, *waiting_buffers_tail;
86 
87  int64_t packets_sent;
89  int64_t frames_output;
91  int eos_sent;
96 
97 // Assume decoder is guaranteed to produce output after at least this many
98 // packets (where each packet contains 1 frame).
99 #define MAX_DELAYED_FRAMES 16
100 
102 {
103  if (ref &&
104  atomic_fetch_add_explicit(&ref->refcount, -1, memory_order_acq_rel) == 1) {
105  mmal_pool_destroy(ref->pool);
106  av_free(ref);
107  }
108 }
109 
110 static void ffmmal_release_frame(void *opaque, uint8_t *data)
111 {
112  FFBufferRef *ref = (void *)data;
113 
114  mmal_buffer_header_release(ref->buffer);
116 
117  av_free(ref);
118 }
119 
120 // Setup frame with a new reference to buffer. The buffer must have been
121 // allocated from the given pool.
123  MMAL_BUFFER_HEADER_T *buffer)
124 {
125  FFBufferRef *ref = av_mallocz(sizeof(*ref));
126  if (!ref)
127  return AVERROR(ENOMEM);
128 
129  ref->pool = pool;
130  ref->buffer = buffer;
131 
132  frame->buf[0] = av_buffer_create((void *)ref, sizeof(*ref),
135  if (!frame->buf[0]) {
136  av_free(ref);
137  return AVERROR(ENOMEM);
138  }
139 
140  atomic_fetch_add_explicit(&ref->pool->refcount, 1, memory_order_relaxed);
141  mmal_buffer_header_acquire(buffer);
142 
143  frame->format = AV_PIX_FMT_MMAL;
144  frame->data[3] = (uint8_t *)ref->buffer;
145  return 0;
146 }
147 
149 {
150  MMALDecodeContext *ctx = avctx->priv_data;
151  MMAL_COMPONENT_T *decoder = ctx->decoder;
152  MMAL_BUFFER_HEADER_T *buffer;
153 
154  mmal_port_disable(decoder->input[0]);
155  mmal_port_disable(decoder->output[0]);
156  mmal_port_disable(decoder->control);
157 
158  mmal_port_flush(decoder->input[0]);
159  mmal_port_flush(decoder->output[0]);
160  mmal_port_flush(decoder->control);
161 
162  while ((buffer = mmal_queue_get(ctx->queue_decoded_frames)))
163  mmal_buffer_header_release(buffer);
164 
165  while (ctx->waiting_buffers) {
166  FFBufferEntry *buffer = ctx->waiting_buffers;
167 
168  ctx->waiting_buffers = buffer->next;
169 
170  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
172 
173  av_buffer_unref(&buffer->ref);
174  av_free(buffer);
175  }
176  ctx->waiting_buffers_tail = NULL;
177 
179 
180  ctx->frames_output = ctx->eos_received = ctx->eos_sent = ctx->packets_sent = ctx->extradata_sent = 0;
181 }
182 
184 {
185  MMALDecodeContext *ctx = avctx->priv_data;
186 
187  if (ctx->decoder)
188  ffmmal_stop_decoder(avctx);
189 
190  mmal_component_destroy(ctx->decoder);
191  ctx->decoder = NULL;
192  mmal_queue_destroy(ctx->queue_decoded_frames);
193  mmal_pool_destroy(ctx->pool_in);
195 
196  mmal_vc_deinit();
197 
198  return 0;
199 }
200 
201 static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
202 {
203  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
204  MMALDecodeContext *ctx = avctx->priv_data;
205 
206  if (!buffer->cmd) {
207  FFBufferEntry *entry = buffer->user_data;
208  av_buffer_unref(&entry->ref);
209  if (entry->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
210  atomic_fetch_add(&ctx->packets_buffered, -1);
211  av_free(entry);
212  }
213  mmal_buffer_header_release(buffer);
214 }
215 
216 static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
217 {
218  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
219  MMALDecodeContext *ctx = avctx->priv_data;
220 
221  mmal_queue_put(ctx->queue_decoded_frames, buffer);
222 }
223 
224 static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
225 {
226  AVCodecContext *avctx = (AVCodecContext*)port->userdata;
227  MMAL_STATUS_T status;
228 
229  if (buffer->cmd == MMAL_EVENT_ERROR) {
230  status = *(uint32_t *)buffer->data;
231  av_log(avctx, AV_LOG_ERROR, "MMAL error %d on control port\n", (int)status);
232  } else {
233  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on control port\n",
234  av_fourcc2str(buffer->cmd));
235  }
236 
237  mmal_buffer_header_release(buffer);
238 }
239 
240 // Feed free output buffers to the decoder.
242 {
243  MMALDecodeContext *ctx = avctx->priv_data;
244  MMAL_BUFFER_HEADER_T *buffer;
245  MMAL_STATUS_T status;
246 
247  if (!ctx->pool_out)
248  return AVERROR_UNKNOWN; // format change code failed with OOM previously
249 
250  while ((buffer = mmal_queue_get(ctx->pool_out->pool->queue))) {
251  if ((status = mmal_port_send_buffer(ctx->decoder->output[0], buffer))) {
252  mmal_buffer_header_release(buffer);
253  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending output buffer.\n", (int)status);
254  return AVERROR_UNKNOWN;
255  }
256  }
257 
258  return 0;
259 }
260 
261 static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
262 {
263  switch (fourcc) {
264  case MMAL_COLOR_SPACE_BT470_2_BG:
265  case MMAL_COLOR_SPACE_BT470_2_M:
266  case MMAL_COLOR_SPACE_ITUR_BT601: return AVCOL_SPC_BT470BG;
267  case MMAL_COLOR_SPACE_ITUR_BT709: return AVCOL_SPC_BT709;
268  case MMAL_COLOR_SPACE_FCC: return AVCOL_SPC_FCC;
269  case MMAL_COLOR_SPACE_SMPTE240M: return AVCOL_SPC_SMPTE240M;
270  default: return AVCOL_SPC_UNSPECIFIED;
271  }
272 }
273 
275 {
276  MMALDecodeContext *ctx = avctx->priv_data;
277  MMAL_STATUS_T status;
278  int ret = 0;
279  MMAL_COMPONENT_T *decoder = ctx->decoder;
280  MMAL_ES_FORMAT_T *format_out = decoder->output[0]->format;
281  MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T interlace_type;
282 
284  if (!(ctx->pool_out = av_mallocz(sizeof(*ctx->pool_out)))) {
285  ret = AVERROR(ENOMEM);
286  goto fail;
287  }
288  atomic_init(&ctx->pool_out->refcount, 1);
289 
290  if (!format_out)
291  goto fail;
292 
293  if ((status = mmal_port_parameter_set_uint32(decoder->output[0], MMAL_PARAMETER_EXTRA_BUFFERS, ctx->extra_buffers)))
294  goto fail;
295 
296  if ((status = mmal_port_parameter_set_boolean(decoder->output[0], MMAL_PARAMETER_VIDEO_INTERPOLATE_TIMESTAMPS, 0)))
297  goto fail;
298 
299  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
300  format_out->encoding = MMAL_ENCODING_OPAQUE;
301  } else {
302  format_out->encoding_variant = format_out->encoding = MMAL_ENCODING_I420;
303  }
304 
305  if ((status = mmal_port_format_commit(decoder->output[0])))
306  goto fail;
307 
308  interlace_type.hdr.id = MMAL_PARAMETER_VIDEO_INTERLACE_TYPE;
309  interlace_type.hdr.size = sizeof(MMAL_PARAMETER_VIDEO_INTERLACE_TYPE_T);
310  status = mmal_port_parameter_get(decoder->output[0], &interlace_type.hdr);
311  if (status != MMAL_SUCCESS) {
312  av_log(avctx, AV_LOG_ERROR, "Cannot read MMAL interlace information!\n");
313  } else {
314  ctx->interlaced_frame = (interlace_type.eMode != MMAL_InterlaceProgressive);
315  ctx->top_field_first = (interlace_type.eMode == MMAL_InterlaceFieldsInterleavedUpperFirst);
316  }
317 
318  if ((ret = ff_set_dimensions(avctx, format_out->es->video.crop.x + format_out->es->video.crop.width,
319  format_out->es->video.crop.y + format_out->es->video.crop.height)) < 0)
320  goto fail;
321 
322  if (format_out->es->video.par.num && format_out->es->video.par.den) {
323  avctx->sample_aspect_ratio.num = format_out->es->video.par.num;
324  avctx->sample_aspect_ratio.den = format_out->es->video.par.den;
325  }
326  if (format_out->es->video.frame_rate.num && format_out->es->video.frame_rate.den) {
327  avctx->framerate.num = format_out->es->video.frame_rate.num;
328  avctx->framerate.den = format_out->es->video.frame_rate.den;
329  }
330 
331  avctx->colorspace = ffmmal_csp_to_av_csp(format_out->es->video.color_space);
332 
333  decoder->output[0]->buffer_size =
334  FFMAX(decoder->output[0]->buffer_size_min, decoder->output[0]->buffer_size_recommended);
335  decoder->output[0]->buffer_num =
336  FFMAX(decoder->output[0]->buffer_num_min, decoder->output[0]->buffer_num_recommended) + ctx->extra_buffers;
337  ctx->pool_out->pool = mmal_pool_create(decoder->output[0]->buffer_num,
338  decoder->output[0]->buffer_size);
339  if (!ctx->pool_out->pool) {
340  ret = AVERROR(ENOMEM);
341  goto fail;
342  }
343 
344  return 0;
345 
346 fail:
347  return ret < 0 ? ret : AVERROR_UNKNOWN;
348 }
349 
351 {
352  MMALDecodeContext *ctx = avctx->priv_data;
353  MMAL_STATUS_T status;
354  MMAL_ES_FORMAT_T *format_in;
355  MMAL_COMPONENT_T *decoder;
356  int ret = 0;
357 
358  bcm_host_init();
359 
360  if (mmal_vc_init()) {
361  av_log(avctx, AV_LOG_ERROR, "Cannot initialize MMAL VC driver!\n");
362  return AVERROR(ENOSYS);
363  }
364 
365  if ((ret = ff_get_format(avctx, avctx->codec->pix_fmts)) < 0)
366  return ret;
367 
368  avctx->pix_fmt = ret;
369 
370  if ((status = mmal_component_create(MMAL_COMPONENT_DEFAULT_VIDEO_DECODER, &ctx->decoder)))
371  goto fail;
372 
373  decoder = ctx->decoder;
374 
375  format_in = decoder->input[0]->format;
376  format_in->type = MMAL_ES_TYPE_VIDEO;
377  switch (avctx->codec_id) {
379  format_in->encoding = MMAL_ENCODING_MP2V;
380  break;
381  case AV_CODEC_ID_MPEG4:
382  format_in->encoding = MMAL_ENCODING_MP4V;
383  break;
384  case AV_CODEC_ID_VC1:
385  format_in->encoding = MMAL_ENCODING_WVC1;
386  break;
387  case AV_CODEC_ID_H264:
388  default:
389  format_in->encoding = MMAL_ENCODING_H264;
390  break;
391  }
392  format_in->es->video.width = FFALIGN(avctx->width, 32);
393  format_in->es->video.height = FFALIGN(avctx->height, 16);
394  format_in->es->video.crop.width = avctx->width;
395  format_in->es->video.crop.height = avctx->height;
396  format_in->es->video.frame_rate.num = 24000;
397  format_in->es->video.frame_rate.den = 1001;
398  format_in->es->video.par.num = avctx->sample_aspect_ratio.num;
399  format_in->es->video.par.den = avctx->sample_aspect_ratio.den;
400  format_in->flags = MMAL_ES_FORMAT_FLAG_FRAMED;
401 
402  av_log(avctx, AV_LOG_DEBUG, "Using MMAL %s encoding.\n",
403  av_fourcc2str(format_in->encoding));
404 
405 #if HAVE_MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS
406  if (mmal_port_parameter_set_uint32(decoder->input[0], MMAL_PARAMETER_VIDEO_MAX_NUM_CALLBACKS,
407  -1 - ctx->extra_decoder_buffers)) {
408  av_log(avctx, AV_LOG_WARNING, "Could not set input buffering limit.\n");
409  }
410 #endif
411 
412  if ((status = mmal_port_format_commit(decoder->input[0])))
413  goto fail;
414 
415  decoder->input[0]->buffer_num =
416  FFMAX(decoder->input[0]->buffer_num_min, 20);
417  decoder->input[0]->buffer_size =
418  FFMAX(decoder->input[0]->buffer_size_min, 512 * 1024);
419  ctx->pool_in = mmal_pool_create(decoder->input[0]->buffer_num, 0);
420  if (!ctx->pool_in) {
421  ret = AVERROR(ENOMEM);
422  goto fail;
423  }
424 
425  if ((ret = ffmal_update_format(avctx)) < 0)
426  goto fail;
427 
428  ctx->queue_decoded_frames = mmal_queue_create();
429  if (!ctx->queue_decoded_frames)
430  goto fail;
431 
432  decoder->input[0]->userdata = (void*)avctx;
433  decoder->output[0]->userdata = (void*)avctx;
434  decoder->control->userdata = (void*)avctx;
435 
436  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
437  goto fail;
438  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
439  goto fail;
440  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
441  goto fail;
442 
443  if ((status = mmal_component_enable(decoder)))
444  goto fail;
445 
446  return 0;
447 
448 fail:
449  ffmmal_close_decoder(avctx);
450  return ret < 0 ? ret : AVERROR_UNKNOWN;
451 }
452 
453 static void ffmmal_flush(AVCodecContext *avctx)
454 {
455  MMALDecodeContext *ctx = avctx->priv_data;
456  MMAL_COMPONENT_T *decoder = ctx->decoder;
457  MMAL_STATUS_T status;
458 
459  ffmmal_stop_decoder(avctx);
460 
461  if ((status = mmal_port_enable(decoder->control, control_port_cb)))
462  goto fail;
463  if ((status = mmal_port_enable(decoder->input[0], input_callback)))
464  goto fail;
465  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
466  goto fail;
467 
468  return;
469 
470 fail:
471  av_log(avctx, AV_LOG_ERROR, "MMAL flush error: %i\n", (int)status);
472 }
473 
474 // Split packets and add them to the waiting_buffers list. We don't queue them
475 // immediately, because it can happen that the decoder is temporarily blocked
476 // (due to us not reading/returning enough output buffers) and won't accept
477 // new input. (This wouldn't be an issue if MMAL input buffers always were
478 // complete frames - then the input buffer just would have to be big enough.)
479 // If is_extradata is set, send it as MMAL_BUFFER_HEADER_FLAG_CONFIG.
480 static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt,
481  int is_extradata)
482 {
483  MMALDecodeContext *ctx = avctx->priv_data;
484  AVBufferRef *buf = NULL;
485  int size = 0;
486  uint8_t *data = (uint8_t *)"";
487  uint8_t *start;
488  int ret = 0;
489 
490  if (avpkt->size) {
491  if (avpkt->buf) {
492  buf = av_buffer_ref(avpkt->buf);
493  size = avpkt->size;
494  data = avpkt->data;
495  } else {
496  buf = av_buffer_alloc(avpkt->size);
497  if (buf) {
498  memcpy(buf->data, avpkt->data, avpkt->size);
499  size = buf->size;
500  data = buf->data;
501  }
502  }
503  if (!buf) {
504  ret = AVERROR(ENOMEM);
505  goto done;
506  }
507  if (!is_extradata)
508  ctx->packets_sent++;
509  } else {
510  if (ctx->eos_sent)
511  goto done;
512  if (!ctx->packets_sent) {
513  // Short-cut the flush logic to avoid upsetting MMAL.
514  ctx->eos_sent = 1;
515  ctx->eos_received = 1;
516  goto done;
517  }
518  }
519 
520  start = data;
521 
522  do {
523  FFBufferEntry *buffer = av_mallocz(sizeof(*buffer));
524  if (!buffer) {
525  ret = AVERROR(ENOMEM);
526  goto done;
527  }
528 
529  buffer->data = data;
530  buffer->length = FFMIN(size, ctx->decoder->input[0]->buffer_size);
531 
532  if (is_extradata)
533  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_CONFIG;
534 
535  if (data == start)
536  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_START;
537 
538  data += buffer->length;
539  size -= buffer->length;
540 
541  buffer->pts = avpkt->pts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->pts;
542  buffer->dts = avpkt->dts == AV_NOPTS_VALUE ? MMAL_TIME_UNKNOWN : avpkt->dts;
543 
544  if (!size) {
545  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_FRAME_END;
547  }
548 
549  if (!buffer->length) {
550  buffer->flags |= MMAL_BUFFER_HEADER_FLAG_EOS;
551  ctx->eos_sent = 1;
552  }
553 
554  if (buf) {
555  buffer->ref = av_buffer_ref(buf);
556  if (!buffer->ref) {
557  av_free(buffer);
558  ret = AVERROR(ENOMEM);
559  goto done;
560  }
561  }
562 
563  // Insert at end of the list
564  if (!ctx->waiting_buffers)
565  ctx->waiting_buffers = buffer;
566  if (ctx->waiting_buffers_tail)
569  } while (size);
570 
571 done:
572  av_buffer_unref(&buf);
573  return ret;
574 }
575 
576 // Move prepared/split packets from waiting_buffers to the MMAL decoder.
578 {
579  MMALDecodeContext *ctx = avctx->priv_data;
580 
581  while (ctx->waiting_buffers) {
582  MMAL_BUFFER_HEADER_T *mbuffer;
584  MMAL_STATUS_T status;
585 
586  mbuffer = mmal_queue_get(ctx->pool_in->queue);
587  if (!mbuffer)
588  return 0;
589 
590  buffer = ctx->waiting_buffers;
591 
592  mmal_buffer_header_reset(mbuffer);
593  mbuffer->cmd = 0;
594  mbuffer->pts = buffer->pts;
595  mbuffer->dts = buffer->dts;
596  mbuffer->flags = buffer->flags;
597  mbuffer->data = buffer->data;
598  mbuffer->length = buffer->length;
599  mbuffer->user_data = buffer;
600  mbuffer->alloc_size = ctx->decoder->input[0]->buffer_size;
601 
602  // Remove from start of the list
603  ctx->waiting_buffers = buffer->next;
604  if (ctx->waiting_buffers_tail == buffer)
605  ctx->waiting_buffers_tail = NULL;
606 
607  if ((status = mmal_port_send_buffer(ctx->decoder->input[0], mbuffer))) {
608  mmal_buffer_header_release(mbuffer);
609  av_buffer_unref(&buffer->ref);
610  if (buffer->flags & MMAL_BUFFER_HEADER_FLAG_FRAME_END)
612  av_free(buffer);
613  }
614 
615  if (status) {
616  av_log(avctx, AV_LOG_ERROR, "MMAL error %d when sending input\n", (int)status);
617  return AVERROR_UNKNOWN;
618  }
619  }
620 
621  return 0;
622 }
623 
625  MMAL_BUFFER_HEADER_T *buffer)
626 {
627  MMALDecodeContext *ctx = avctx->priv_data;
628  int ret = 0;
629 
630  frame->interlaced_frame = ctx->interlaced_frame;
631  frame->top_field_first = ctx->top_field_first;
632 
633  if (avctx->pix_fmt == AV_PIX_FMT_MMAL) {
634  if (!ctx->pool_out)
635  return AVERROR_UNKNOWN; // format change code failed with OOM previously
636 
637  if ((ret = ff_decode_frame_props(avctx, frame)) < 0)
638  goto done;
639 
640  if ((ret = ffmmal_set_ref(frame, ctx->pool_out, buffer)) < 0)
641  goto done;
642  } else {
643  int w = FFALIGN(avctx->width, 32);
644  int h = FFALIGN(avctx->height, 16);
645  uint8_t *src[4];
646  int linesize[4];
647 
648  if ((ret = ff_get_buffer(avctx, frame, 0)) < 0)
649  goto done;
650 
651  av_image_fill_arrays(src, linesize,
652  buffer->data + buffer->type->video.offset[0],
653  avctx->pix_fmt, w, h, 1);
654  av_image_copy(frame->data, frame->linesize, src, linesize,
655  avctx->pix_fmt, avctx->width, avctx->height);
656  }
657 
658  frame->pts = buffer->pts == MMAL_TIME_UNKNOWN ? AV_NOPTS_VALUE : buffer->pts;
659 #if FF_API_PKT_PTS
661  frame->pkt_pts = frame->pts;
663 #endif
664  frame->pkt_dts = AV_NOPTS_VALUE;
665 
666 done:
667  return ret;
668 }
669 
670 // Fetch a decoded buffer and place it into the frame parameter.
671 static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
672 {
673  MMALDecodeContext *ctx = avctx->priv_data;
674  MMAL_BUFFER_HEADER_T *buffer = NULL;
675  MMAL_STATUS_T status = 0;
676  int ret = 0;
677 
678  if (ctx->eos_received)
679  goto done;
680 
681  while (1) {
682  // To ensure decoding in lockstep with a constant delay between fed packets
683  // and output frames, we always wait until an output buffer is available.
684  // Except during start we don't know after how many input packets the decoder
685  // is going to return the first buffer, and we can't distinguish decoder
686  // being busy from decoder waiting for input. So just poll at the start and
687  // keep feeding new data to the buffer.
688  // We are pretty sure the decoder will produce output if we sent more input
689  // frames than what a H.264 decoder could logically delay. This avoids too
690  // excessive buffering.
691  // We also wait if we sent eos, but didn't receive it yet (think of decoding
692  // stream with a very low number of frames).
694  (ctx->packets_sent && ctx->eos_sent)) {
695  // MMAL will ignore broken input packets, which means the frame we
696  // expect here may never arrive. Dealing with this correctly is
697  // complicated, so here's a hack to avoid that it freezes forever
698  // in this unlikely situation.
699  buffer = mmal_queue_timedwait(ctx->queue_decoded_frames, 100);
700  if (!buffer) {
701  av_log(avctx, AV_LOG_ERROR, "Did not get output frame from MMAL.\n");
702  ret = AVERROR_UNKNOWN;
703  goto done;
704  }
705  } else {
706  buffer = mmal_queue_get(ctx->queue_decoded_frames);
707  if (!buffer)
708  goto done;
709  }
710 
711  ctx->eos_received |= !!(buffer->flags & MMAL_BUFFER_HEADER_FLAG_EOS);
712  if (ctx->eos_received)
713  goto done;
714 
715  if (buffer->cmd == MMAL_EVENT_FORMAT_CHANGED) {
716  MMAL_COMPONENT_T *decoder = ctx->decoder;
717  MMAL_EVENT_FORMAT_CHANGED_T *ev = mmal_event_format_changed_get(buffer);
718  MMAL_BUFFER_HEADER_T *stale_buffer;
719 
720  av_log(avctx, AV_LOG_INFO, "Changing output format.\n");
721 
722  if ((status = mmal_port_disable(decoder->output[0])))
723  goto done;
724 
725  while ((stale_buffer = mmal_queue_get(ctx->queue_decoded_frames)))
726  mmal_buffer_header_release(stale_buffer);
727 
728  mmal_format_copy(decoder->output[0]->format, ev->format);
729 
730  if ((ret = ffmal_update_format(avctx)) < 0)
731  goto done;
732 
733  if ((status = mmal_port_enable(decoder->output[0], output_callback)))
734  goto done;
735 
736  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
737  goto done;
738 
739  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
740  goto done;
741 
742  mmal_buffer_header_release(buffer);
743  continue;
744  } else if (buffer->cmd) {
745  av_log(avctx, AV_LOG_WARNING, "Unknown MMAL event %s on output port\n",
746  av_fourcc2str(buffer->cmd));
747  goto done;
748  } else if (buffer->length == 0) {
749  // Unused output buffer that got drained after format change.
750  mmal_buffer_header_release(buffer);
751  continue;
752  }
753 
754  ctx->frames_output++;
755 
756  if ((ret = ffmal_copy_frame(avctx, frame, buffer)) < 0)
757  goto done;
758 
759  *got_frame = 1;
760  break;
761  }
762 
763 done:
764  if (buffer)
765  mmal_buffer_header_release(buffer);
766  if (status && ret >= 0)
767  ret = AVERROR_UNKNOWN;
768  return ret;
769 }
770 
771 static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame,
772  AVPacket *avpkt)
773 {
774  MMALDecodeContext *ctx = avctx->priv_data;
775  AVFrame *frame = data;
776  int ret = 0;
777 
778  if (avctx->extradata_size && !ctx->extradata_sent) {
779  AVPacket pkt = {0};
780  av_init_packet(&pkt);
781  pkt.data = avctx->extradata;
782  pkt.size = avctx->extradata_size;
783  ctx->extradata_sent = 1;
784  if ((ret = ffmmal_add_packet(avctx, &pkt, 1)) < 0)
785  return ret;
786  }
787 
788  if ((ret = ffmmal_add_packet(avctx, avpkt, 0)) < 0)
789  return ret;
790 
791  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
792  return ret;
793 
794  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
795  return ret;
796 
797  if ((ret = ffmmal_read_frame(avctx, frame, got_frame)) < 0)
798  return ret;
799 
800  // ffmmal_read_frame() can block for a while. Since the decoder is
801  // asynchronous, it's a good idea to fill the ports again.
802 
803  if ((ret = ffmmal_fill_output_port(avctx)) < 0)
804  return ret;
805 
806  if ((ret = ffmmal_fill_input_port(avctx)) < 0)
807  return ret;
808 
809  return ret;
810 }
811 
813  HW_CONFIG_INTERNAL(MMAL),
814  NULL
815 };
816 
817 static const AVOption options[]={
818  {"extra_buffers", "extra buffers", offsetof(MMALDecodeContext, extra_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
819  {"extra_decoder_buffers", "extra MMAL internal buffered frames", offsetof(MMALDecodeContext, extra_decoder_buffers), AV_OPT_TYPE_INT, {.i64 = 10}, 0, 256, 0},
820  {NULL}
821 };
822 
823 #define FFMMAL_DEC_CLASS(NAME) \
824  static const AVClass ffmmal_##NAME##_dec_class = { \
825  .class_name = "mmal_" #NAME "_dec", \
826  .item_name = av_default_item_name, \
827  .option = options, \
828  .version = LIBAVUTIL_VERSION_INT, \
829  };
830 
831 #define FFMMAL_DEC(NAME, ID) \
832  FFMMAL_DEC_CLASS(NAME) \
833  AVCodec ff_##NAME##_mmal_decoder = { \
834  .name = #NAME "_mmal", \
835  .long_name = NULL_IF_CONFIG_SMALL(#NAME " (mmal)"), \
836  .type = AVMEDIA_TYPE_VIDEO, \
837  .id = ID, \
838  .priv_data_size = sizeof(MMALDecodeContext), \
839  .init = ffmmal_init_decoder, \
840  .close = ffmmal_close_decoder, \
841  .decode = ffmmal_decode, \
842  .flush = ffmmal_flush, \
843  .priv_class = &ffmmal_##NAME##_dec_class, \
844  .capabilities = AV_CODEC_CAP_DELAY | AV_CODEC_CAP_HARDWARE, \
845  .caps_internal = FF_CODEC_CAP_SETS_PKT_DTS, \
846  .pix_fmts = (const enum AVPixelFormat[]) { AV_PIX_FMT_MMAL, \
847  AV_PIX_FMT_YUV420P, \
848  AV_PIX_FMT_NONE}, \
849  .hw_configs = mmal_hw_configs, \
850  .wrapper_name = "mmal", \
851  };
852 
also ITU-R BT1361 / IEC 61966-2-4 xvYCC709 / SMPTE RP177 Annex B
Definition: pixfmt.h:514
#define NULL
Definition: coverity.c:32
int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt)
Select the (possibly hardware accelerated) pixel format.
Definition: decode.c:1292
const struct AVCodec * codec
Definition: avcodec.h:535
AVRational framerate
Definition: avcodec.h:2069
MMAL_POOL_T * pool
Definition: mmaldec.c:61
void av_buffer_unref(AVBufferRef **buf)
Free a given reference and automatically free the buffer if there are no more references to it...
Definition: buffer.c:125
This structure describes decoded (raw) audio or video data.
Definition: frame.h:308
#define FFMMAL_DEC(NAME, ID)
Definition: mmaldec.c:831
AVOption.
Definition: opt.h:248
ptrdiff_t const GLvoid * data
Definition: opengl_enc.c:100
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:200
int ff_set_dimensions(AVCodecContext *s, int width, int height)
Check that the provided frame dimensions are valid and set them on the codec context.
Definition: utils.c:106
AVBufferRef * buf[AV_NUM_DATA_POINTERS]
AVBuffer references backing the data for this frame.
Definition: frame.h:499
atomic_int packets_buffered
Definition: mmaldec.c:88
also ITU-R BT601-6 625 / ITU-R BT1358 625 / ITU-R BT1700 625 PAL & SECAM / IEC 61966-2-4 xvYCC601 ...
Definition: pixfmt.h:518
FFPoolRef * pool
Definition: mmaldec.c:66
uint32_t fourcc
Definition: vaapi_decode.c:239
int num
Numerator.
Definition: rational.h:59
int size
Definition: packet.h:364
int av_image_fill_arrays(uint8_t *dst_data[4], int dst_linesize[4], const uint8_t *src, enum AVPixelFormat pix_fmt, int width, int height, int align)
Setup the data pointers and linesizes based on the specified image parameters and the provided array...
Definition: imgutils.c:446
AVRational sample_aspect_ratio
sample aspect ratio (0 if unknown) That is the width of a pixel divided by the height of the pixel...
Definition: avcodec.h:905
enum AVPixelFormat pix_fmt
Pixel format, see AV_PIX_FMT_xxx.
Definition: avcodec.h:736
static int ffmmal_fill_output_port(AVCodecContext *avctx)
Definition: mmaldec.c:241
void * av_mallocz(size_t size)
Allocate a memory block with alignment suitable for all memory accesses (including vectors if availab...
Definition: mem.c:237
int ff_decode_frame_props(AVCodecContext *avctx, AVFrame *frame)
Set various frame properties from the codec context / packet data.
Definition: decode.c:1697
static AVPacket pkt
atomic_int refcount
Definition: mmaldec.c:60
intptr_t atomic_int
Definition: stdatomic.h:55
int64_t dts
Definition: mmaldec.c:51
#define HW_CONFIG_INTERNAL(format)
Definition: hwconfig.h:57
static int ffmmal_fill_input_port(AVCodecContext *avctx)
Definition: mmaldec.c:577
static int ffmmal_decode(AVCodecContext *avctx, void *data, int *got_frame, AVPacket *avpkt)
Definition: mmaldec.c:771
functionally identical to above
Definition: pixfmt.h:520
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
uint8_t
#define av_cold
Definition: attributes.h:88
AVOptions.
MMAL_BUFFER_HEADER_T * buffer
Definition: mmaldec.c:65
AVColorSpace
YUV colorspace type.
Definition: pixfmt.h:512
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:401
uint8_t * extradata
some codecs need / can use extradata like Huffman tables.
Definition: avcodec.h:627
static void ffmmal_flush(AVCodecContext *avctx)
Definition: mmaldec.c:453
uint8_t * data
Definition: packet.h:363
static av_cold int ffmmal_init_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:350
static const AVOption options[]
Definition: mmaldec.c:817
static av_cold int ffmmal_close_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:183
int interlaced_frame
The content of the picture is interlaced.
Definition: frame.h:455
#define AV_BUFFER_FLAG_READONLY
Always treat the buffer as read-only, even when it has only one reference.
Definition: buffer.h:113
MMAL_POOL_T * pool_in
Definition: mmaldec.c:76
ptrdiff_t size
Definition: opengl_enc.c:100
int top_field_first
Definition: mmaldec.c:94
#define FFALIGN(x, a)
Definition: macros.h:48
#define av_log(a,...)
#define src
Definition: vp8dsp.c:254
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:194
static void output_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:216
#define atomic_load(object)
Definition: stdatomic.h:93
int extradata_sent
Definition: mmaldec.c:92
static void ffmmal_poolref_unref(FFPoolRef *ref)
Definition: mmaldec.c:101
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:215
#define av_fourcc2str(fourcc)
Definition: avutil.h:348
AVBufferRef * buf
A reference to the reference-counted buffer where the packet data is stored.
Definition: packet.h:346
simple assert() macros that are a bit more flexible than ISO C assert().
static int ffmmal_set_ref(AVFrame *frame, FFPoolRef *pool, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:122
FFBufferEntry * waiting_buffers_tail
Definition: mmaldec.c:85
AVBufferRef * av_buffer_create(uint8_t *data, int size, void(*free)(void *opaque, uint8_t *data), void *opaque, int flags)
Create an AVBuffer from an existing array.
Definition: buffer.c:29
int extra_decoder_buffers
Definition: mmaldec.c:72
#define FFMAX(a, b)
Definition: common.h:94
#define fail()
Definition: checkasm.h:123
static void control_port_cb(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:224
void av_image_copy(uint8_t *dst_data[4], int dst_linesizes[4], const uint8_t *src_data[4], const int src_linesizes[4], enum AVPixelFormat pix_fmt, int width, int height)
Copy image in src_data to dst_data.
Definition: imgutils.c:422
AVBufferRef * ref
Definition: mmaldec.c:48
size_t length
Definition: mmaldec.c:50
static int ffmal_copy_frame(AVCodecContext *avctx, AVFrame *frame, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:624
AVClass * av_class
Definition: mmaldec.c:70
static enum AVColorSpace ffmmal_csp_to_av_csp(MMAL_FOURCC_T fourcc)
Definition: mmaldec.c:261
enum AVPixelFormat * pix_fmts
array of supported pixel formats, or NULL if unknown, array is terminated by -1
Definition: codec.h:211
FFPoolRef * pool_out
Definition: mmaldec.c:77
#define FFMIN(a, b)
Definition: common.h:96
FFBufferEntry * waiting_buffers
Definition: mmaldec.c:85
int64_t frames_output
Definition: mmaldec.c:89
static const chunk_decoder decoder[8]
Definition: dfa.c:330
int width
picture width / height.
Definition: avcodec.h:699
uint8_t w
Definition: llviddspenc.c:38
AVFormatContext * ctx
Definition: movenc.c:48
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
#define MAX_DELAYED_FRAMES
Definition: mmaldec.c:99
preferred ID for MPEG-1/2 video decoding
Definition: codec_id.h:51
FCC Title 47 Code of Federal Regulations 73.682 (a)(20)
Definition: pixfmt.h:517
static void ffmmal_stop_decoder(AVCodecContext *avctx)
Definition: mmaldec.c:148
MMAL_QUEUE_T * queue_decoded_frames
Definition: mmaldec.c:75
if(ret)
#define atomic_fetch_add_explicit(object, operand, order)
Definition: stdatomic.h:149
int format
format of the frame, -1 if unknown or unset Values correspond to enum AVPixelFormat for video frames...
Definition: frame.h:381
#define atomic_fetch_add(object, operand)
Definition: stdatomic.h:131
int interlaced_frame
Definition: mmaldec.c:93
#define AV_LOG_INFO
Standard information.
Definition: log.h:205
struct FFBufferEntry * next
Definition: mmaldec.c:53
Libavcodec external API header.
enum AVCodecID codec_id
Definition: avcodec.h:536
AVBufferRef * av_buffer_alloc(int size)
Allocate an AVBuffer of the given size using av_malloc().
Definition: buffer.c:67
int linesize[AV_NUM_DATA_POINTERS]
For video, size in bytes of each picture line.
Definition: frame.h:339
main external API structure.
Definition: avcodec.h:526
uint8_t * data
The data buffer.
Definition: buffer.h:89
void * data
Definition: mmaldec.c:49
static void input_callback(MMAL_PORT_T *port, MMAL_BUFFER_HEADER_T *buffer)
Definition: mmaldec.c:201
int ff_get_buffer(AVCodecContext *avctx, AVFrame *frame, int flags)
Get a buffer for a frame.
Definition: decode.c:1872
int extradata_size
Definition: avcodec.h:628
Describe the class of an AVClass context structure.
Definition: log.h:67
enum AVColorSpace colorspace
YUV colorspace type.
Definition: avcodec.h:1154
they must not be accessed directly The fifo field contains the frames that are queued in the input for processing by the filter The status_in and status_out fields contains the queued status(EOF or error) of the link
refcounted data buffer API
int size
Size of data in bytes.
Definition: buffer.h:93
static int ffmmal_read_frame(AVCodecContext *avctx, AVFrame *frame, int *got_frame)
Definition: mmaldec.c:671
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:322
attribute_deprecated int64_t pkt_pts
PTS copied from the AVPacket that was decoded to produce this frame.
Definition: frame.h:409
int64_t packets_sent
Definition: mmaldec.c:87
int64_t pkt_dts
DTS copied from the AVPacket that triggered returning this frame.
Definition: frame.h:417
A reference to a data buffer.
Definition: buffer.h:81
#define FF_DISABLE_DEPRECATION_WARNINGS
Definition: internal.h:84
common internal api header.
common internal and external API header
static int ffmal_update_format(AVCodecContext *avctx)
Definition: mmaldec.c:274
AVBufferRef * av_buffer_ref(AVBufferRef *buf)
Create a new reference to an AVBuffer.
Definition: buffer.c:93
void av_init_packet(AVPacket *pkt)
Initialize optional fields of a packet with default values.
Definition: avpacket.c:35
int den
Denominator.
Definition: rational.h:60
#define AVERROR_UNKNOWN
Unknown error, typically from an external library.
Definition: error.h:71
static void ffmmal_release_frame(void *opaque, uint8_t *data)
Definition: mmaldec.c:110
void * priv_data
Definition: avcodec.h:553
#define av_free(p)
#define FF_ENABLE_DEPRECATION_WARNINGS
Definition: internal.h:85
int top_field_first
If the content is interlaced, is top field displayed first.
Definition: frame.h:460
static int ffmmal_add_packet(AVCodecContext *avctx, AVPacket *avpkt, int is_extradata)
Definition: mmaldec.c:480
int64_t dts
Decompression timestamp in AVStream->time_base units; the time at which the packet is decompressed...
Definition: packet.h:362
#define atomic_init(obj, value)
Definition: stdatomic.h:33
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
HW acceleration though MMAL, data[3] contains a pointer to the MMAL_BUFFER_HEADER_T structure...
Definition: pixfmt.h:227
This structure stores compressed data.
Definition: packet.h:340
MMAL_COMPONENT_T * decoder
Definition: mmaldec.c:74
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: packet.h:356
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:248
static const AVCodecHWConfigInternal * mmal_hw_configs[]
Definition: mmaldec.c:812
GLuint buffer
Definition: opengl_enc.c:101
int64_t pts
Definition: mmaldec.c:51