FFmpeg
rtpdec_hevc.c
Go to the documentation of this file.
1 /*
2  * RTP parser for HEVC/H.265 payload format (draft version 6)
3  * Copyright (c) 2014 Thomas Volkert <thomas@homer-conferencing.com>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/avassert.h"
23 #include "libavutil/avstring.h"
24 #include "libavutil/mem.h"
25 
26 #include "avformat.h"
27 #include "internal.h"
28 #include "rtpdec.h"
29 #include "rtpdec_formats.h"
30 
31 #define RTP_HEVC_PAYLOAD_HEADER_SIZE 2
32 #define RTP_HEVC_FU_HEADER_SIZE 1
33 #define RTP_HEVC_DONL_FIELD_SIZE 2
34 #define RTP_HEVC_DOND_FIELD_SIZE 1
35 #define RTP_HEVC_AP_NALU_LENGTH_FIELD_SIZE 2
36 #define HEVC_SPECIFIED_NAL_UNIT_TYPES 48
37 
38 /* SDP out-of-band signaling data */
39 struct PayloadContext {
42  uint8_t *sps, *pps, *vps, *sei;
44 };
45 
46 static const uint8_t start_sequence[] = { 0x00, 0x00, 0x00, 0x01 };
47 
49  AVStream *stream,
50  PayloadContext *hevc_data,
51  const char *attr, const char *value)
52 {
53  /* profile-space: 0-3 */
54  /* profile-id: 0-31 */
55  if (!strcmp(attr, "profile-id")) {
56  hevc_data->profile_id = atoi(value);
57  av_log(s, AV_LOG_TRACE, "SDP: found profile-id: %d\n", hevc_data->profile_id);
58  }
59 
60  /* tier-flag: 0-1 */
61  /* level-id: 0-255 */
62  /* interop-constraints: [base16] */
63  /* profile-compatibility-indicator: [base16] */
64  /* sprop-sub-layer-id: 0-6, defines highest possible value for TID, default: 6 */
65  /* recv-sub-layer-id: 0-6 */
66  /* max-recv-level-id: 0-255 */
67  /* tx-mode: MSM,SSM */
68  /* sprop-vps: [base64] */
69  /* sprop-sps: [base64] */
70  /* sprop-pps: [base64] */
71  /* sprop-sei: [base64] */
72  if (!strcmp(attr, "sprop-vps") || !strcmp(attr, "sprop-sps") ||
73  !strcmp(attr, "sprop-pps") || !strcmp(attr, "sprop-sei")) {
74  uint8_t **data_ptr = NULL;
75  int *size_ptr = NULL;
76  if (!strcmp(attr, "sprop-vps")) {
77  data_ptr = &hevc_data->vps;
78  size_ptr = &hevc_data->vps_size;
79  } else if (!strcmp(attr, "sprop-sps")) {
80  data_ptr = &hevc_data->sps;
81  size_ptr = &hevc_data->sps_size;
82  } else if (!strcmp(attr, "sprop-pps")) {
83  data_ptr = &hevc_data->pps;
84  size_ptr = &hevc_data->pps_size;
85  } else if (!strcmp(attr, "sprop-sei")) {
86  data_ptr = &hevc_data->sei;
87  size_ptr = &hevc_data->sei_size;
88  } else
89  av_assert0(0);
90 
92  size_ptr, value);
93  }
94 
95  /* max-lsr, max-lps, max-cpb, max-dpb, max-br, max-tr, max-tc */
96  /* max-fps */
97 
98  /* sprop-max-don-diff: 0-32767
99 
100  When the RTP stream depends on one or more other RTP
101  streams (in this case tx-mode MUST be equal to "MSM" and
102  MSM is in use), this parameter MUST be present and the
103  value MUST be greater than 0.
104  */
105  if (!strcmp(attr, "sprop-max-don-diff")) {
106  if (atoi(value) > 0)
107  hevc_data->using_donl_field = 1;
108  av_log(s, AV_LOG_TRACE, "Found sprop-max-don-diff in SDP, DON field usage is: %d\n",
109  hevc_data->using_donl_field);
110  }
111 
112  /* sprop-depack-buf-nalus: 0-32767 */
113  if (!strcmp(attr, "sprop-depack-buf-nalus")) {
114  if (atoi(value) > 0)
115  hevc_data->using_donl_field = 1;
116  av_log(s, AV_LOG_TRACE, "Found sprop-depack-buf-nalus in SDP, DON field usage is: %d\n",
117  hevc_data->using_donl_field);
118  }
119 
120  /* sprop-depack-buf-bytes: 0-4294967295 */
121  /* depack-buf-cap */
122  /* sprop-segmentation-id: 0-3 */
123  /* sprop-spatial-segmentation-idc: [base16] */
124  /* dec-parallel-ca: */
125  /* include-dph */
126 
127  return 0;
128 }
129 
130 static av_cold int hevc_parse_sdp_line(AVFormatContext *ctx, int st_index,
131  PayloadContext *hevc_data, const char *line)
132 {
133  AVStream *current_stream;
134  AVCodecParameters *par;
135  const char *sdp_line_ptr = line;
136 
137  if (st_index < 0)
138  return 0;
139 
140  current_stream = ctx->streams[st_index];
141  par = current_stream->codecpar;
142 
143  if (av_strstart(sdp_line_ptr, "framesize:", &sdp_line_ptr)) {
144  ff_h264_parse_framesize(par, sdp_line_ptr);
145  } else if (av_strstart(sdp_line_ptr, "fmtp:", &sdp_line_ptr)) {
146  int ret = ff_parse_fmtp(ctx, current_stream, hevc_data, sdp_line_ptr,
148  if (hevc_data->vps_size || hevc_data->sps_size ||
149  hevc_data->pps_size || hevc_data->sei_size) {
150  par->extradata_size = hevc_data->vps_size + hevc_data->sps_size +
151  hevc_data->pps_size + hevc_data->sei_size;
152  if ((ret = ff_alloc_extradata(par, par->extradata_size)) >= 0) {
153  int pos = 0;
154  memcpy(par->extradata + pos, hevc_data->vps, hevc_data->vps_size);
155  pos += hevc_data->vps_size;
156  memcpy(par->extradata + pos, hevc_data->sps, hevc_data->sps_size);
157  pos += hevc_data->sps_size;
158  memcpy(par->extradata + pos, hevc_data->pps, hevc_data->pps_size);
159  pos += hevc_data->pps_size;
160  memcpy(par->extradata + pos, hevc_data->sei, hevc_data->sei_size);
161  }
162 
163  av_freep(&hevc_data->vps);
164  av_freep(&hevc_data->sps);
165  av_freep(&hevc_data->pps);
166  av_freep(&hevc_data->sei);
167  hevc_data->vps_size = 0;
168  hevc_data->sps_size = 0;
169  hevc_data->pps_size = 0;
170  hevc_data->sei_size = 0;
171  }
172  return ret;
173  }
174 
175  return 0;
176 }
177 
179  AVStream *st, AVPacket *pkt, uint32_t *timestamp,
180  const uint8_t *buf, int len, uint16_t seq,
181  int flags)
182 {
183  const uint8_t *rtp_pl = buf;
184  int tid, lid, nal_type;
185  int first_fragment, last_fragment, fu_type;
186  uint8_t new_nal_header[2];
187  int res = 0;
188 
189  /* sanity check for size of input packet: 1 byte payload at least */
190  if (len < RTP_HEVC_PAYLOAD_HEADER_SIZE + 1) {
191  av_log(ctx, AV_LOG_ERROR, "Too short RTP/HEVC packet, got %d bytes\n", len);
192  return AVERROR_INVALIDDATA;
193  }
194 
195  /*
196  * decode the HEVC payload header according to section 4 of draft version 6:
197  *
198  * 0 1
199  * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
200  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
201  * |F| Type | LayerId | TID |
202  * +-------------+-----------------+
203  *
204  * Forbidden zero (F): 1 bit
205  * NAL unit type (Type): 6 bits
206  * NUH layer ID (LayerId): 6 bits
207  * NUH temporal ID plus 1 (TID): 3 bits
208  */
209  nal_type = (buf[0] >> 1) & 0x3f;
210  lid = ((buf[0] << 5) & 0x20) | ((buf[1] >> 3) & 0x1f);
211  tid = buf[1] & 0x07;
212 
213  /* sanity check for correct layer ID */
214  if (lid) {
215  /* future scalable or 3D video coding extensions */
216  avpriv_report_missing_feature(ctx, "Multi-layer HEVC coding");
217  return AVERROR_PATCHWELCOME;
218  }
219 
220  /* sanity check for correct temporal ID */
221  if (!tid) {
222  av_log(ctx, AV_LOG_ERROR, "Illegal temporal ID in RTP/HEVC packet\n");
223  return AVERROR_INVALIDDATA;
224  }
225 
226  /* sanity check for correct NAL unit type */
227  if (nal_type > 50) {
228  av_log(ctx, AV_LOG_ERROR, "Unsupported (HEVC) NAL type (%d)\n", nal_type);
229  return AVERROR_INVALIDDATA;
230  }
231 
232  switch (nal_type) {
233  /* video parameter set (VPS) */
234  case 32:
235  /* sequence parameter set (SPS) */
236  case 33:
237  /* picture parameter set (PPS) */
238  case 34:
239  /* supplemental enhancement information (SEI) */
240  case 39:
241  /* single NAL unit packet */
242  default:
243  /* create A/V packet */
244  if ((res = av_new_packet(pkt, sizeof(start_sequence) + len)) < 0)
245  return res;
246  /* A/V packet: copy start sequence */
247  memcpy(pkt->data, start_sequence, sizeof(start_sequence));
248  /* A/V packet: copy NAL unit data */
249  memcpy(pkt->data + sizeof(start_sequence), buf, len);
250 
251  break;
252  /* aggregated packet (AP) - with two or more NAL units */
253  case 48:
254  /* pass the HEVC payload header */
257 
258  /* pass the HEVC DONL field */
259  if (rtp_hevc_ctx->using_donl_field) {
262  }
263 
264  res = ff_h264_handle_aggregated_packet(ctx, rtp_hevc_ctx, pkt, buf, len,
265  rtp_hevc_ctx->using_donl_field ?
267  NULL, 0);
268  if (res < 0)
269  return res;
270  break;
271  /* fragmentation unit (FU) */
272  case 49:
273  /* pass the HEVC payload header */
276 
277  /*
278  * decode the FU header
279  *
280  * 0 1 2 3 4 5 6 7
281  * +-+-+-+-+-+-+-+-+
282  * |S|E| FuType |
283  * +---------------+
284  *
285  * Start fragment (S): 1 bit
286  * End fragment (E): 1 bit
287  * FuType: 6 bits
288  */
289  first_fragment = buf[0] & 0x80;
290  last_fragment = buf[0] & 0x40;
291  fu_type = buf[0] & 0x3f;
292 
293  /* pass the HEVC FU header */
296 
297  /* pass the HEVC DONL field */
298  if (rtp_hevc_ctx->using_donl_field) {
301  }
302 
303  av_log(ctx, AV_LOG_TRACE, " FU type %d with %d bytes\n", fu_type, len);
304 
305  /* sanity check for size of input packet: 1 byte payload at least */
306  if (len <= 0) {
307  if (len < 0) {
309  "Too short RTP/HEVC packet, got %d bytes of NAL unit type %d\n",
310  len, nal_type);
311  return AVERROR_INVALIDDATA;
312  } else {
313  return AVERROR(EAGAIN);
314  }
315  }
316 
317  if (first_fragment && last_fragment) {
318  av_log(ctx, AV_LOG_ERROR, "Illegal combination of S and E bit in RTP/HEVC packet\n");
319  return AVERROR_INVALIDDATA;
320  }
321 
322  new_nal_header[0] = (rtp_pl[0] & 0x81) | (fu_type << 1);
323  new_nal_header[1] = rtp_pl[1];
324 
325  res = ff_h264_handle_frag_packet(pkt, buf, len, first_fragment,
326  new_nal_header, sizeof(new_nal_header));
327 
328  break;
329  /* PACI packet */
330  case 50:
331  /* Temporal scalability control information (TSCI) */
332  avpriv_report_missing_feature(ctx, "PACI packets for RTP/HEVC");
333  res = AVERROR_PATCHWELCOME;
334  break;
335  }
336 
337  pkt->stream_index = st->index;
338 
339  return res;
340 }
341 
343  .enc_name = "H265",
344  .codec_type = AVMEDIA_TYPE_VIDEO,
345  .codec_id = AV_CODEC_ID_HEVC,
346  .need_parsing = AVSTREAM_PARSE_FULL,
347  .priv_data_size = sizeof(PayloadContext),
348  .parse_sdp_a_line = hevc_parse_sdp_line,
350 };
AVCodecParameters::extradata
uint8_t * extradata
Extra binary data needed for initializing the decoder, codec-dependent.
Definition: codec_par.h:69
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
AVCodecParameters
This struct describes the properties of an encoded stream.
Definition: codec_par.h:47
rtpdec_formats.h
ff_parse_fmtp
int ff_parse_fmtp(AVFormatContext *s, AVStream *stream, PayloadContext *data, const char *p, int(*parse_fmtp)(AVFormatContext *s, AVStream *stream, PayloadContext *data, const char *attr, const char *value))
Definition: rtpdec.c:965
RTP_HEVC_PAYLOAD_HEADER_SIZE
#define RTP_HEVC_PAYLOAD_HEADER_SIZE
Definition: rtpdec_hevc.c:31
AVFormatContext::streams
AVStream ** streams
A list of all streams in the file.
Definition: avformat.h:1323
AVPacket::data
uint8_t * data
Definition: packet.h:520
PayloadContext::sps_size
int sps_size
Definition: rtpdec_hevc.c:43
PayloadContext::pps
uint8_t * pps
Definition: rtpdec_hevc.c:42
hevc_handle_packet
static int hevc_handle_packet(AVFormatContext *ctx, PayloadContext *rtp_hevc_ctx, AVStream *st, AVPacket *pkt, uint32_t *timestamp, const uint8_t *buf, int len, uint16_t seq, int flags)
Definition: rtpdec_hevc.c:178
RTP_HEVC_FU_HEADER_SIZE
#define RTP_HEVC_FU_HEADER_SIZE
Definition: rtpdec_hevc.c:32
RTPDynamicProtocolHandler::enc_name
const char * enc_name
Definition: rtpdec.h:117
PayloadContext::profile_id
int profile_id
Definition: rtpdec_hevc.c:41
start_sequence
static const uint8_t start_sequence[]
Definition: rtpdec_hevc.c:46
avassert.h
AV_LOG_TRACE
#define AV_LOG_TRACE
Extremely verbose debugging, useful for libav* development.
Definition: log.h:206
pkt
AVPacket * pkt
Definition: movenc.c:60
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
hevc_parse_sdp_line
static av_cold int hevc_parse_sdp_line(AVFormatContext *ctx, int st_index, PayloadContext *hevc_data, const char *line)
Definition: rtpdec_hevc.c:130
s
#define s(width, name)
Definition: cbs_vp9.c:198
av_new_packet
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: packet.c:98
PayloadContext::sps
uint8_t * sps
Definition: rtpdec_hevc.c:42
RTP_HEVC_DONL_FIELD_SIZE
#define RTP_HEVC_DONL_FIELD_SIZE
Definition: rtpdec_hevc.c:33
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:40
ctx
AVFormatContext * ctx
Definition: movenc.c:49
ff_h264_parse_framesize
void ff_h264_parse_framesize(AVCodecParameters *par, const char *p)
Definition: rtpdec_h264.c:185
PayloadContext::vps_size
int vps_size
Definition: rtpdec_hevc.c:43
AVFormatContext
Format I/O context.
Definition: avformat.h:1255
internal.h
AVStream::codecpar
AVCodecParameters * codecpar
Codec parameters associated with this stream.
Definition: avformat.h:766
NULL
#define NULL
Definition: coverity.c:32
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
rtpdec.h
AVCodecParameters::extradata_size
int extradata_size
Size of the extradata content in bytes.
Definition: codec_par.h:73
PayloadContext::sei
uint8_t * sei
Definition: rtpdec_hevc.c:42
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
line
Definition: graph2dot.c:48
av_strstart
int av_strstart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str.
Definition: avstring.c:36
ff_h264_handle_frag_packet
int ff_h264_handle_frag_packet(AVPacket *pkt, const uint8_t *buf, int len, int start_bit, const uint8_t *nal_header, int nal_header_len)
Definition: rtpdec_h264.c:265
ff_hevc_dynamic_handler
const RTPDynamicProtocolHandler ff_hevc_dynamic_handler
Definition: rtpdec_hevc.c:342
AV_CODEC_ID_HEVC
@ AV_CODEC_ID_HEVC
Definition: codec_id.h:226
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
PayloadContext::using_donl_field
int using_donl_field
Definition: rtpdec_hevc.c:40
len
int len
Definition: vorbis_enc_data.h:426
RTP_HEVC_DOND_FIELD_SIZE
#define RTP_HEVC_DOND_FIELD_SIZE
Definition: rtpdec_hevc.c:34
hevc_sdp_parse_fmtp_config
static av_cold int hevc_sdp_parse_fmtp_config(AVFormatContext *s, AVStream *stream, PayloadContext *hevc_data, const char *attr, const char *value)
Definition: rtpdec_hevc.c:48
ret
ret
Definition: filter_design.txt:187
AVStream
Stream structure.
Definition: avformat.h:743
pos
unsigned int pos
Definition: spdifenc.c:414
avformat.h
PayloadContext::vps
uint8_t * vps
Definition: rtpdec_hevc.c:42
AVStream::index
int index
stream index in AVFormatContext
Definition: avformat.h:749
AVPacket::stream_index
int stream_index
Definition: packet.h:522
parse_packet
static int parse_packet(AVFormatContext *s, AVPacket *pkt, int stream_index, int flush)
Parse a packet, add all split parts to parse_queue.
Definition: demux.c:1155
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
mem.h
ff_h264_handle_aggregated_packet
int ff_h264_handle_aggregated_packet(AVFormatContext *ctx, PayloadContext *data, AVPacket *pkt, const uint8_t *buf, int len, int start_skip, int *nal_counters, int nal_mask)
Definition: rtpdec_h264.c:207
PayloadContext::sei_size
int sei_size
Definition: rtpdec_hevc.c:43
AVPacket
This structure stores compressed data.
Definition: packet.h:497
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
AVSTREAM_PARSE_FULL
@ AVSTREAM_PARSE_FULL
full parsing and repack
Definition: avformat.h:593
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:474
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AVERROR_INVALIDDATA
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:61
avstring.h
PayloadContext
RTP/JPEG specific private data.
Definition: rdt.c:85
PayloadContext::pps_size
int pps_size
Definition: rtpdec_hevc.c:43
ff_h264_parse_sprop_parameter_sets
int ff_h264_parse_sprop_parameter_sets(AVFormatContext *s, uint8_t **data_ptr, int *size_ptr, const char *value)
Definition: rtpdec_h264.c:97
line
The official guide to swscale for confused that consecutive non overlapping rectangles of slice_bottom special converter These generally are unscaled converters of common like for each output line the vertical scaler pulls lines from a ring buffer When the ring buffer does not contain the wanted line
Definition: swscale.txt:40
RTPDynamicProtocolHandler
Definition: rtpdec.h:116
ff_alloc_extradata
int ff_alloc_extradata(AVCodecParameters *par, int size)
Allocate extradata with additional AV_INPUT_BUFFER_PADDING_SIZE at end which is always set to 0.
Definition: utils.c:227