FFmpeg
 All Data Structures Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
libstagefright.cpp
Go to the documentation of this file.
1 /*
2  * Interface to the Android Stagefright library for
3  * H/W accelerated H.264 decoding
4  *
5  * Copyright (C) 2011 Mohamed Naufal
6  * Copyright (C) 2011 Martin Storsjö
7  *
8  * This file is part of FFmpeg.
9  *
10  * FFmpeg is free software; you can redistribute it and/or
11  * modify it under the terms of the GNU Lesser General Public
12  * License as published by the Free Software Foundation; either
13  * version 2.1 of the License, or (at your option) any later version.
14  *
15  * FFmpeg is distributed in the hope that it will be useful,
16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18  * Lesser General Public License for more details.
19  *
20  * You should have received a copy of the GNU Lesser General Public
21  * License along with FFmpeg; if not, write to the Free Software
22  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
23  */
24 
25 #include <binder/ProcessState.h>
26 #include <media/stagefright/MetaData.h>
27 #include <media/stagefright/MediaBufferGroup.h>
28 #include <media/stagefright/MediaDebug.h>
29 #include <media/stagefright/MediaDefs.h>
30 #include <media/stagefright/OMXClient.h>
31 #include <media/stagefright/OMXCodec.h>
32 #include <utils/List.h>
33 #include <new>
34 #include <map>
35 
36 extern "C" {
37 #include "avcodec.h"
38 #include "libavutil/imgutils.h"
39 }
40 
41 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
42 
43 using namespace android;
44 
45 struct Frame {
46  status_t status;
47  size_t size;
48  int64_t time;
49  int key;
52 };
53 
54 struct TimeStamp {
55  int64_t pts;
57 };
58 
59 class CustomSource;
60 
66  sp<MediaSource> *source;
67  List<Frame*> *in_queue, *out_queue;
71 
74  volatile sig_atomic_t thread_started, thread_exited, stop_decode;
75 
77  std::map<int64_t, TimeStamp> *ts_map;
78  int64_t frame_index;
79 
82 
83  OMXClient *client;
84  sp<MediaSource> *decoder;
85  const char *decoder_component;
86 };
87 
88 class CustomSource : public MediaSource {
89 public:
90  CustomSource(AVCodecContext *avctx, sp<MetaData> meta) {
91  s = (StagefrightContext*)avctx->priv_data;
92  source_meta = meta;
93  frame_size = (avctx->width * avctx->height * 3) / 2;
94  buf_group.add_buffer(new MediaBuffer(frame_size));
95  }
96 
97  virtual sp<MetaData> getFormat() {
98  return source_meta;
99  }
100 
101  virtual status_t start(MetaData *params) {
102  return OK;
103  }
104 
105  virtual status_t stop() {
106  return OK;
107  }
108 
109  virtual status_t read(MediaBuffer **buffer,
110  const MediaSource::ReadOptions *options) {
111  Frame *frame;
112  status_t ret;
113 
114  if (s->thread_exited)
115  return ERROR_END_OF_STREAM;
116  pthread_mutex_lock(&s->in_mutex);
117 
118  while (s->in_queue->empty())
119  pthread_cond_wait(&s->condition, &s->in_mutex);
120 
121  frame = *s->in_queue->begin();
122  ret = frame->status;
123 
124  if (ret == OK) {
125  ret = buf_group.acquire_buffer(buffer);
126  if (ret == OK) {
127  memcpy((*buffer)->data(), frame->buffer, frame->size);
128  (*buffer)->set_range(0, frame->size);
129  (*buffer)->meta_data()->clear();
130  (*buffer)->meta_data()->setInt32(kKeyIsSyncFrame,frame->key);
131  (*buffer)->meta_data()->setInt64(kKeyTime, frame->time);
132  } else {
133  av_log(s->avctx, AV_LOG_ERROR, "Failed to acquire MediaBuffer\n");
134  }
135  av_freep(&frame->buffer);
136  }
137 
138  s->in_queue->erase(s->in_queue->begin());
139  pthread_mutex_unlock(&s->in_mutex);
140 
141  av_freep(&frame);
142  return ret;
143  }
144 
145 private:
146  MediaBufferGroup buf_group;
147  sp<MetaData> source_meta;
150 };
151 
152 void* decode_thread(void *arg)
153 {
154  AVCodecContext *avctx = (AVCodecContext*)arg;
156  const AVPixFmtDescriptor *pix_desc = av_pix_fmt_desc_get(avctx->pix_fmt);
157  Frame* frame;
158  MediaBuffer *buffer;
159  int32_t w, h;
160  int decode_done = 0;
161  int ret;
162  int src_linesize[3];
163  const uint8_t *src_data[3];
164  int64_t out_frame_index = 0;
165 
166  do {
167  buffer = NULL;
168  frame = (Frame*)av_mallocz(sizeof(Frame));
169  if (!frame) {
170  frame = s->end_frame;
171  frame->status = AVERROR(ENOMEM);
172  decode_done = 1;
173  s->end_frame = NULL;
174  goto push_frame;
175  }
176  frame->status = (*s->decoder)->read(&buffer);
177  if (frame->status == OK) {
178  sp<MetaData> outFormat = (*s->decoder)->getFormat();
179  outFormat->findInt32(kKeyWidth , &w);
180  outFormat->findInt32(kKeyHeight, &h);
181  frame->vframe = (AVFrame*)av_mallocz(sizeof(AVFrame));
182  if (!frame->vframe) {
183  frame->status = AVERROR(ENOMEM);
184  decode_done = 1;
185  buffer->release();
186  goto push_frame;
187  }
188  ret = ff_get_buffer(avctx, frame->vframe);
189  if (ret < 0) {
190  av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n");
191  frame->status = ret;
192  decode_done = 1;
193  buffer->release();
194  goto push_frame;
195  }
196 
197  // The OMX.SEC decoder doesn't signal the modified width/height
198  if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) &&
199  (w & 15 || h & 15)) {
200  if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == buffer->range_length()) {
201  w = (w + 15)&~15;
202  h = (h + 15)&~15;
203  }
204  }
205 
206  if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) {
207  avctx->width = w;
208  avctx->height = h;
209  }
210 
211  src_linesize[0] = av_image_get_linesize(avctx->pix_fmt, w, 0);
212  src_linesize[1] = av_image_get_linesize(avctx->pix_fmt, w, 1);
213  src_linesize[2] = av_image_get_linesize(avctx->pix_fmt, w, 2);
214 
215  src_data[0] = (uint8_t*)buffer->data();
216  src_data[1] = src_data[0] + src_linesize[0] * h;
217  src_data[2] = src_data[1] + src_linesize[1] * -(-h>>pix_desc->log2_chroma_h);
218  av_image_copy(frame->vframe->data, frame->vframe->linesize,
219  src_data, src_linesize,
220  avctx->pix_fmt, avctx->width, avctx->height);
221 
222  buffer->meta_data()->findInt64(kKeyTime, &out_frame_index);
223  if (out_frame_index && s->ts_map->count(out_frame_index) > 0) {
224  frame->vframe->pts = (*s->ts_map)[out_frame_index].pts;
225  frame->vframe->reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque;
226  s->ts_map->erase(out_frame_index);
227  }
228  buffer->release();
229  } else if (frame->status == INFO_FORMAT_CHANGED) {
230  if (buffer)
231  buffer->release();
232  av_free(frame);
233  continue;
234  } else {
235  decode_done = 1;
236  }
237 push_frame:
238  while (true) {
240  if (s->out_queue->size() >= 10) {
242  usleep(10000);
243  continue;
244  }
245  break;
246  }
247  s->out_queue->push_back(frame);
249  } while (!decode_done && !s->stop_decode);
250 
251  s->thread_exited = true;
252 
253  return 0;
254 }
255 
257 {
259  sp<MetaData> meta, outFormat;
260  int32_t colorFormat = 0;
261  int ret;
262 
263  if (!avctx->extradata || !avctx->extradata_size || avctx->extradata[0] != 1)
264  return -1;
265 
266  s->avctx = avctx;
267  s->bsfc = av_bitstream_filter_init("h264_mp4toannexb");
268  if (!s->bsfc) {
269  av_log(avctx, AV_LOG_ERROR, "Cannot open the h264_mp4toannexb BSF!\n");
270  return -1;
271  }
272 
276  if (!s->orig_extradata) {
277  ret = AVERROR(ENOMEM);
278  goto fail;
279  }
280  memcpy(s->orig_extradata, avctx->extradata, avctx->extradata_size);
281 
282  meta = new MetaData;
283  if (meta == NULL) {
284  ret = AVERROR(ENOMEM);
285  goto fail;
286  }
287  meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_VIDEO_AVC);
288  meta->setInt32(kKeyWidth, avctx->width);
289  meta->setInt32(kKeyHeight, avctx->height);
290  meta->setData(kKeyAVCC, kTypeAVCC, avctx->extradata, avctx->extradata_size);
291 
292  android::ProcessState::self()->startThreadPool();
293 
294  s->source = new sp<MediaSource>();
295  *s->source = new CustomSource(avctx, meta);
296  s->in_queue = new List<Frame*>;
297  s->out_queue = new List<Frame*>;
298  s->ts_map = new std::map<int64_t, TimeStamp>;
299  s->client = new OMXClient;
300  s->end_frame = (Frame*)av_mallocz(sizeof(Frame));
301  if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client ||
302  !s->ts_map || !s->end_frame) {
303  ret = AVERROR(ENOMEM);
304  goto fail;
305  }
306 
307  if (s->client->connect() != OK) {
308  av_log(avctx, AV_LOG_ERROR, "Cannot connect OMX client\n");
309  ret = -1;
310  goto fail;
311  }
312 
313  s->decoder = new sp<MediaSource>();
314  *s->decoder = OMXCodec::Create(s->client->interface(), meta,
315  false, *s->source, NULL,
316  OMXCodec::kClientNeedsFramebuffer);
317  if ((*s->decoder)->start() != OK) {
318  av_log(avctx, AV_LOG_ERROR, "Cannot start decoder\n");
319  ret = -1;
320  s->client->disconnect();
321  goto fail;
322  }
323 
324  outFormat = (*s->decoder)->getFormat();
325  outFormat->findInt32(kKeyColorFormat, &colorFormat);
326  if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar ||
327  colorFormat == OMX_COLOR_FormatYUV420SemiPlanar)
328  avctx->pix_fmt = AV_PIX_FMT_NV21;
329  else if (colorFormat == OMX_COLOR_FormatYCbYCr)
330  avctx->pix_fmt = AV_PIX_FMT_YUYV422;
331  else if (colorFormat == OMX_COLOR_FormatCbYCrY)
332  avctx->pix_fmt = AV_PIX_FMT_UYVY422;
333  else
334  avctx->pix_fmt = AV_PIX_FMT_YUV420P;
335 
336  outFormat->findCString(kKeyDecoderComponent, &s->decoder_component);
337  if (s->decoder_component)
339 
343  return 0;
344 
345 fail:
348  av_freep(&s->end_frame);
349  delete s->in_queue;
350  delete s->out_queue;
351  delete s->ts_map;
352  delete s->client;
353  return ret;
354 }
355 
357  int *got_frame, AVPacket *avpkt)
358 {
360  Frame *frame;
361  status_t status;
362  int orig_size = avpkt->size;
363  AVPacket pkt = *avpkt;
364  AVFrame *ret_frame;
365 
366  if (!s->thread_started) {
368  s->thread_started = true;
369  }
370 
371  if (avpkt && avpkt->data) {
373  avpkt->data, avpkt->size, avpkt->flags & AV_PKT_FLAG_KEY);
374  avpkt = &pkt;
375  }
376 
377  if (!s->source_done) {
378  if(!s->dummy_buf) {
379  s->dummy_buf = (uint8_t*)av_malloc(avpkt->size);
380  if (!s->dummy_buf)
381  return AVERROR(ENOMEM);
382  s->dummy_bufsize = avpkt->size;
383  memcpy(s->dummy_buf, avpkt->data, avpkt->size);
384  }
385 
386  frame = (Frame*)av_mallocz(sizeof(Frame));
387  if (avpkt->data) {
388  frame->status = OK;
389  frame->size = avpkt->size;
390  frame->key = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0;
391  frame->buffer = (uint8_t*)av_malloc(avpkt->size);
392  if (!frame->buffer) {
393  av_freep(&frame);
394  return AVERROR(ENOMEM);
395  }
396  uint8_t *ptr = avpkt->data;
397  // The OMX.SEC decoder fails without this.
398  if (avpkt->size == orig_size + avctx->extradata_size) {
399  ptr += avctx->extradata_size;
400  frame->size = orig_size;
401  }
402  memcpy(frame->buffer, ptr, orig_size);
403  if (avpkt == &pkt)
404  av_free(avpkt->data);
405 
406  frame->time = ++s->frame_index;
407  (*s->ts_map)[s->frame_index].pts = avpkt->pts;
408  (*s->ts_map)[s->frame_index].reordered_opaque = avctx->reordered_opaque;
409  } else {
410  frame->status = ERROR_END_OF_STREAM;
411  s->source_done = true;
412  }
413 
414  while (true) {
415  if (s->thread_exited) {
416  s->source_done = true;
417  break;
418  }
420  if (s->in_queue->size() >= 10) {
422  usleep(10000);
423  continue;
424  }
425  s->in_queue->push_back(frame);
428  break;
429  }
430  }
431  while (true) {
433  if (!s->out_queue->empty()) break;
435  if (s->source_done) {
436  usleep(10000);
437  continue;
438  } else {
439  return orig_size;
440  }
441  }
442 
443  frame = *s->out_queue->begin();
444  s->out_queue->erase(s->out_queue->begin());
446 
447  ret_frame = frame->vframe;
448  status = frame->status;
449  av_freep(&frame);
450 
451  if (status == ERROR_END_OF_STREAM)
452  return 0;
453  if (status != OK) {
454  if (status == AVERROR(ENOMEM))
455  return status;
456  av_log(avctx, AV_LOG_ERROR, "Decode failed: %x\n", status);
457  return -1;
458  }
459 
460  if (s->prev_frame) {
461  avctx->release_buffer(avctx, s->prev_frame);
462  av_freep(&s->prev_frame);
463  }
464  s->prev_frame = ret_frame;
465 
466  *got_frame = 1;
467  *(AVFrame*)data = *ret_frame;
468  return orig_size;
469 }
470 
472 {
474  Frame *frame;
475 
476  if (s->thread_started) {
477  if (!s->thread_exited) {
478  s->stop_decode = 1;
479 
480  // Make sure decode_thread() doesn't get stuck
482  while (!s->out_queue->empty()) {
483  frame = *s->out_queue->begin();
484  s->out_queue->erase(s->out_queue->begin());
485  if (frame->vframe) {
486  avctx->release_buffer(avctx, frame->vframe);
487  av_freep(&frame->vframe);
488  }
489  av_freep(&frame);
490  }
492 
493  // Feed a dummy frame prior to signalling EOF.
494  // This is required to terminate the decoder(OMX.SEC)
495  // when only one frame is read during stream info detection.
496  if (s->dummy_buf && (frame = (Frame*)av_mallocz(sizeof(Frame)))) {
497  frame->status = OK;
498  frame->size = s->dummy_bufsize;
499  frame->key = 1;
500  frame->buffer = s->dummy_buf;
502  s->in_queue->push_back(frame);
505  s->dummy_buf = NULL;
506  }
507 
509  s->end_frame->status = ERROR_END_OF_STREAM;
510  s->in_queue->push_back(s->end_frame);
513  s->end_frame = NULL;
514  }
515 
517 
518  if (s->prev_frame) {
519  avctx->release_buffer(avctx, s->prev_frame);
520  av_freep(&s->prev_frame);
521  }
522 
523  s->thread_started = false;
524  }
525 
526  while (!s->in_queue->empty()) {
527  frame = *s->in_queue->begin();
528  s->in_queue->erase(s->in_queue->begin());
529  if (frame->size)
530  av_freep(&frame->buffer);
531  av_freep(&frame);
532  }
533 
534  while (!s->out_queue->empty()) {
535  frame = *s->out_queue->begin();
536  s->out_queue->erase(s->out_queue->begin());
537  if (frame->vframe) {
538  avctx->release_buffer(avctx, frame->vframe);
539  av_freep(&frame->vframe);
540  }
541  av_freep(&frame);
542  }
543 
544  (*s->decoder)->stop();
545  s->client->disconnect();
546 
547  if (s->decoder_component)
549  av_freep(&s->dummy_buf);
550  av_freep(&s->end_frame);
551 
552  // Reset the extradata back to the original mp4 format, so that
553  // the next invocation (both when decoding and when called from
554  // av_find_stream_info) get the original mp4 format extradata.
555  av_freep(&avctx->extradata);
556  avctx->extradata = s->orig_extradata;
558 
559  delete s->in_queue;
560  delete s->out_queue;
561  delete s->ts_map;
562  delete s->client;
563  delete s->decoder;
564  delete s->source;
565 
570  return 0;
571 }
572 
574  "libstagefright_h264",
575  NULL_IF_CONFIG_SMALL("libstagefright H.264"),
579  NULL, //supported_framerates
580  NULL, //pix_fmts
581  NULL, //supported_samplerates
582  NULL, //sample_fmts
583  NULL, //channel_layouts
584  0, //max_lowres
585  NULL, //priv_class
586  NULL, //profiles
587  sizeof(StagefrightContext),
588  NULL, //next
589  NULL, //init_thread_copy
590  NULL, //update_thread_context
591  NULL, //defaults
592  NULL, //init_static_data
594  NULL, //encode
595  NULL, //encode2
598 };