FFmpeg
rtpdec_hevc.c
Go to the documentation of this file.
1 /*
2  * RTP parser for HEVC/H.265 payload format (draft version 6)
3  * Copyright (c) 2014 Thomas Volkert <thomas@homer-conferencing.com>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 #include "libavutil/avassert.h"
23 #include "libavutil/avstring.h"
24 
25 #include "avformat.h"
26 #include "internal.h"
27 #include "rtpdec.h"
28 #include "rtpdec_formats.h"
29 
30 #define RTP_HEVC_PAYLOAD_HEADER_SIZE 2
31 #define RTP_HEVC_FU_HEADER_SIZE 1
32 #define RTP_HEVC_DONL_FIELD_SIZE 2
33 #define RTP_HEVC_DOND_FIELD_SIZE 1
34 #define RTP_HEVC_AP_NALU_LENGTH_FIELD_SIZE 2
35 #define HEVC_SPECIFIED_NAL_UNIT_TYPES 48
36 
37 /* SDP out-of-band signaling data */
38 struct PayloadContext {
41  uint8_t *sps, *pps, *vps, *sei;
43 };
44 
45 static const uint8_t start_sequence[] = { 0x00, 0x00, 0x00, 0x01 };
46 
48  AVStream *stream,
49  PayloadContext *hevc_data,
50  const char *attr, const char *value)
51 {
52  /* profile-space: 0-3 */
53  /* profile-id: 0-31 */
54  if (!strcmp(attr, "profile-id")) {
55  hevc_data->profile_id = atoi(value);
56  av_log(s, AV_LOG_TRACE, "SDP: found profile-id: %d\n", hevc_data->profile_id);
57  }
58 
59  /* tier-flag: 0-1 */
60  /* level-id: 0-255 */
61  /* interop-constraints: [base16] */
62  /* profile-compatibility-indicator: [base16] */
63  /* sprop-sub-layer-id: 0-6, defines highest possible value for TID, default: 6 */
64  /* recv-sub-layer-id: 0-6 */
65  /* max-recv-level-id: 0-255 */
66  /* tx-mode: MSM,SSM */
67  /* sprop-vps: [base64] */
68  /* sprop-sps: [base64] */
69  /* sprop-pps: [base64] */
70  /* sprop-sei: [base64] */
71  if (!strcmp(attr, "sprop-vps") || !strcmp(attr, "sprop-sps") ||
72  !strcmp(attr, "sprop-pps") || !strcmp(attr, "sprop-sei")) {
73  uint8_t **data_ptr = NULL;
74  int *size_ptr = NULL;
75  if (!strcmp(attr, "sprop-vps")) {
76  data_ptr = &hevc_data->vps;
77  size_ptr = &hevc_data->vps_size;
78  } else if (!strcmp(attr, "sprop-sps")) {
79  data_ptr = &hevc_data->sps;
80  size_ptr = &hevc_data->sps_size;
81  } else if (!strcmp(attr, "sprop-pps")) {
82  data_ptr = &hevc_data->pps;
83  size_ptr = &hevc_data->pps_size;
84  } else if (!strcmp(attr, "sprop-sei")) {
85  data_ptr = &hevc_data->sei;
86  size_ptr = &hevc_data->sei_size;
87  } else
88  av_assert0(0);
89 
91  size_ptr, value);
92  }
93 
94  /* max-lsr, max-lps, max-cpb, max-dpb, max-br, max-tr, max-tc */
95  /* max-fps */
96 
97  /* sprop-max-don-diff: 0-32767
98 
99  When the RTP stream depends on one or more other RTP
100  streams (in this case tx-mode MUST be equal to "MSM" and
101  MSM is in use), this parameter MUST be present and the
102  value MUST be greater than 0.
103  */
104  if (!strcmp(attr, "sprop-max-don-diff")) {
105  if (atoi(value) > 0)
106  hevc_data->using_donl_field = 1;
107  av_log(s, AV_LOG_TRACE, "Found sprop-max-don-diff in SDP, DON field usage is: %d\n",
108  hevc_data->using_donl_field);
109  }
110 
111  /* sprop-depack-buf-nalus: 0-32767 */
112  if (!strcmp(attr, "sprop-depack-buf-nalus")) {
113  if (atoi(value) > 0)
114  hevc_data->using_donl_field = 1;
115  av_log(s, AV_LOG_TRACE, "Found sprop-depack-buf-nalus in SDP, DON field usage is: %d\n",
116  hevc_data->using_donl_field);
117  }
118 
119  /* sprop-depack-buf-bytes: 0-4294967295 */
120  /* depack-buf-cap */
121  /* sprop-segmentation-id: 0-3 */
122  /* sprop-spatial-segmentation-idc: [base16] */
123  /* dec-parallel-ca: */
124  /* include-dph */
125 
126  return 0;
127 }
128 
129 static av_cold int hevc_parse_sdp_line(AVFormatContext *ctx, int st_index,
130  PayloadContext *hevc_data, const char *line)
131 {
132  AVStream *current_stream;
133  AVCodecParameters *par;
134  const char *sdp_line_ptr = line;
135 
136  if (st_index < 0)
137  return 0;
138 
139  current_stream = ctx->streams[st_index];
140  par = current_stream->codecpar;
141 
142  if (av_strstart(sdp_line_ptr, "framesize:", &sdp_line_ptr)) {
143  ff_h264_parse_framesize(par, sdp_line_ptr);
144  } else if (av_strstart(sdp_line_ptr, "fmtp:", &sdp_line_ptr)) {
145  int ret = ff_parse_fmtp(ctx, current_stream, hevc_data, sdp_line_ptr,
147  if (hevc_data->vps_size || hevc_data->sps_size ||
148  hevc_data->pps_size || hevc_data->sei_size) {
149  par->extradata_size = hevc_data->vps_size + hevc_data->sps_size +
150  hevc_data->pps_size + hevc_data->sei_size;
151  if ((ret = ff_alloc_extradata(par, par->extradata_size)) >= 0) {
152  int pos = 0;
153  memcpy(par->extradata + pos, hevc_data->vps, hevc_data->vps_size);
154  pos += hevc_data->vps_size;
155  memcpy(par->extradata + pos, hevc_data->sps, hevc_data->sps_size);
156  pos += hevc_data->sps_size;
157  memcpy(par->extradata + pos, hevc_data->pps, hevc_data->pps_size);
158  pos += hevc_data->pps_size;
159  memcpy(par->extradata + pos, hevc_data->sei, hevc_data->sei_size);
160  }
161 
162  av_freep(&hevc_data->vps);
163  av_freep(&hevc_data->sps);
164  av_freep(&hevc_data->pps);
165  av_freep(&hevc_data->sei);
166  hevc_data->vps_size = 0;
167  hevc_data->sps_size = 0;
168  hevc_data->pps_size = 0;
169  hevc_data->sei_size = 0;
170  }
171  return ret;
172  }
173 
174  return 0;
175 }
176 
178  AVStream *st, AVPacket *pkt, uint32_t *timestamp,
179  const uint8_t *buf, int len, uint16_t seq,
180  int flags)
181 {
182  const uint8_t *rtp_pl = buf;
183  int tid, lid, nal_type;
184  int first_fragment, last_fragment, fu_type;
185  uint8_t new_nal_header[2];
186  int res = 0;
187 
188  /* sanity check for size of input packet: 1 byte payload at least */
189  if (len < RTP_HEVC_PAYLOAD_HEADER_SIZE + 1) {
190  av_log(ctx, AV_LOG_ERROR, "Too short RTP/HEVC packet, got %d bytes\n", len);
191  return AVERROR_INVALIDDATA;
192  }
193 
194  /*
195  * decode the HEVC payload header according to section 4 of draft version 6:
196  *
197  * 0 1
198  * 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
199  * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
200  * |F| Type | LayerId | TID |
201  * +-------------+-----------------+
202  *
203  * Forbidden zero (F): 1 bit
204  * NAL unit type (Type): 6 bits
205  * NUH layer ID (LayerId): 6 bits
206  * NUH temporal ID plus 1 (TID): 3 bits
207  */
208  nal_type = (buf[0] >> 1) & 0x3f;
209  lid = ((buf[0] << 5) & 0x20) | ((buf[1] >> 3) & 0x1f);
210  tid = buf[1] & 0x07;
211 
212  /* sanity check for correct layer ID */
213  if (lid) {
214  /* future scalable or 3D video coding extensions */
215  avpriv_report_missing_feature(ctx, "Multi-layer HEVC coding");
216  return AVERROR_PATCHWELCOME;
217  }
218 
219  /* sanity check for correct temporal ID */
220  if (!tid) {
221  av_log(ctx, AV_LOG_ERROR, "Illegal temporal ID in RTP/HEVC packet\n");
222  return AVERROR_INVALIDDATA;
223  }
224 
225  /* sanity check for correct NAL unit type */
226  if (nal_type > 50) {
227  av_log(ctx, AV_LOG_ERROR, "Unsupported (HEVC) NAL type (%d)\n", nal_type);
228  return AVERROR_INVALIDDATA;
229  }
230 
231  switch (nal_type) {
232  /* video parameter set (VPS) */
233  case 32:
234  /* sequence parameter set (SPS) */
235  case 33:
236  /* picture parameter set (PPS) */
237  case 34:
238  /* supplemental enhancement information (SEI) */
239  case 39:
240  /* single NAL unit packet */
241  default:
242  /* create A/V packet */
243  if ((res = av_new_packet(pkt, sizeof(start_sequence) + len)) < 0)
244  return res;
245  /* A/V packet: copy start sequence */
246  memcpy(pkt->data, start_sequence, sizeof(start_sequence));
247  /* A/V packet: copy NAL unit data */
248  memcpy(pkt->data + sizeof(start_sequence), buf, len);
249 
250  break;
251  /* aggregated packet (AP) - with two or more NAL units */
252  case 48:
253  /* pass the HEVC payload header */
256 
257  /* pass the HEVC DONL field */
258  if (rtp_hevc_ctx->using_donl_field) {
261  }
262 
263  res = ff_h264_handle_aggregated_packet(ctx, rtp_hevc_ctx, pkt, buf, len,
264  rtp_hevc_ctx->using_donl_field ?
266  NULL, 0);
267  if (res < 0)
268  return res;
269  break;
270  /* fragmentation unit (FU) */
271  case 49:
272  /* pass the HEVC payload header */
275 
276  /*
277  * decode the FU header
278  *
279  * 0 1 2 3 4 5 6 7
280  * +-+-+-+-+-+-+-+-+
281  * |S|E| FuType |
282  * +---------------+
283  *
284  * Start fragment (S): 1 bit
285  * End fragment (E): 1 bit
286  * FuType: 6 bits
287  */
288  first_fragment = buf[0] & 0x80;
289  last_fragment = buf[0] & 0x40;
290  fu_type = buf[0] & 0x3f;
291 
292  /* pass the HEVC FU header */
295 
296  /* pass the HEVC DONL field */
297  if (rtp_hevc_ctx->using_donl_field) {
300  }
301 
302  av_log(ctx, AV_LOG_TRACE, " FU type %d with %d bytes\n", fu_type, len);
303 
304  /* sanity check for size of input packet: 1 byte payload at least */
305  if (len <= 0) {
306  if (len < 0) {
308  "Too short RTP/HEVC packet, got %d bytes of NAL unit type %d\n",
309  len, nal_type);
310  return AVERROR_INVALIDDATA;
311  } else {
312  return AVERROR(EAGAIN);
313  }
314  }
315 
316  if (first_fragment && last_fragment) {
317  av_log(ctx, AV_LOG_ERROR, "Illegal combination of S and E bit in RTP/HEVC packet\n");
318  return AVERROR_INVALIDDATA;
319  }
320 
321  new_nal_header[0] = (rtp_pl[0] & 0x81) | (fu_type << 1);
322  new_nal_header[1] = rtp_pl[1];
323 
324  res = ff_h264_handle_frag_packet(pkt, buf, len, first_fragment,
325  new_nal_header, sizeof(new_nal_header));
326 
327  break;
328  /* PACI packet */
329  case 50:
330  /* Temporal scalability control information (TSCI) */
331  avpriv_report_missing_feature(ctx, "PACI packets for RTP/HEVC");
332  res = AVERROR_PATCHWELCOME;
333  break;
334  }
335 
336  pkt->stream_index = st->index;
337 
338  return res;
339 }
340 
342  .enc_name = "H265",
343  .codec_type = AVMEDIA_TYPE_VIDEO,
344  .codec_id = AV_CODEC_ID_HEVC,
345  .need_parsing = AVSTREAM_PARSE_FULL,
346  .priv_data_size = sizeof(PayloadContext),
347  .parse_sdp_a_line = hevc_parse_sdp_line,
349 };
AVCodecParameters::extradata
uint8_t * extradata
Extra binary data needed for initializing the decoder, codec-dependent.
Definition: codec_par.h:76
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
AVCodecParameters
This struct describes the properties of an encoded stream.
Definition: codec_par.h:54
rtpdec_formats.h
ff_parse_fmtp
int ff_parse_fmtp(AVFormatContext *s, AVStream *stream, PayloadContext *data, const char *p, int(*parse_fmtp)(AVFormatContext *s, AVStream *stream, PayloadContext *data, const char *attr, const char *value))
Definition: rtpdec.c:964
RTP_HEVC_PAYLOAD_HEADER_SIZE
#define RTP_HEVC_PAYLOAD_HEADER_SIZE
Definition: rtpdec_hevc.c:30
AVFormatContext::streams
AVStream ** streams
A list of all streams in the file.
Definition: avformat.h:1284
AVPacket::data
uint8_t * data
Definition: packet.h:374
PayloadContext::sps_size
int sps_size
Definition: rtpdec_hevc.c:42
PayloadContext::pps
uint8_t * pps
Definition: rtpdec_hevc.c:41
hevc_handle_packet
static int hevc_handle_packet(AVFormatContext *ctx, PayloadContext *rtp_hevc_ctx, AVStream *st, AVPacket *pkt, uint32_t *timestamp, const uint8_t *buf, int len, uint16_t seq, int flags)
Definition: rtpdec_hevc.c:177
RTP_HEVC_FU_HEADER_SIZE
#define RTP_HEVC_FU_HEADER_SIZE
Definition: rtpdec_hevc.c:31
RTPDynamicProtocolHandler::enc_name
const char * enc_name
Definition: rtpdec.h:117
PayloadContext::profile_id
int profile_id
Definition: rtpdec_hevc.c:40
start_sequence
static const uint8_t start_sequence[]
Definition: rtpdec_hevc.c:45
avassert.h
AV_LOG_TRACE
#define AV_LOG_TRACE
Extremely verbose debugging, useful for libav* development.
Definition: log.h:206
pkt
AVPacket * pkt
Definition: movenc.c:59
AV_LOG_ERROR
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:180
av_cold
#define av_cold
Definition: attributes.h:90
hevc_parse_sdp_line
static av_cold int hevc_parse_sdp_line(AVFormatContext *ctx, int st_index, PayloadContext *hevc_data, const char *line)
Definition: rtpdec_hevc.c:129
s
#define s(width, name)
Definition: cbs_vp9.c:256
av_new_packet
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:97
PayloadContext::sps
uint8_t * sps
Definition: rtpdec_hevc.c:41
RTP_HEVC_DONL_FIELD_SIZE
#define RTP_HEVC_DONL_FIELD_SIZE
Definition: rtpdec_hevc.c:32
av_assert0
#define av_assert0(cond)
assert() equivalent, that is always enabled.
Definition: avassert.h:37
ctx
AVFormatContext * ctx
Definition: movenc.c:48
ff_h264_parse_framesize
void ff_h264_parse_framesize(AVCodecParameters *par, const char *p)
Definition: rtpdec_h264.c:184
PayloadContext::vps_size
int vps_size
Definition: rtpdec_hevc.c:42
AVFormatContext
Format I/O context.
Definition: avformat.h:1216
internal.h
AVStream::codecpar
AVCodecParameters * codecpar
Codec parameters associated with this stream.
Definition: avformat.h:1108
NULL
#define NULL
Definition: coverity.c:32
AVERROR_PATCHWELCOME
#define AVERROR_PATCHWELCOME
Not yet implemented in FFmpeg, patches welcome.
Definition: error.h:64
rtpdec.h
AVCodecParameters::extradata_size
int extradata_size
Size of the extradata content in bytes.
Definition: codec_par.h:80
PayloadContext::sei
uint8_t * sei
Definition: rtpdec_hevc.c:41
avpriv_report_missing_feature
void avpriv_report_missing_feature(void *avc, const char *msg,...) av_printf_format(2
Log a generic warning message about a missing feature.
line
Definition: graph2dot.c:48
av_strstart
int av_strstart(const char *str, const char *pfx, const char **ptr)
Return non-zero if pfx is a prefix of str.
Definition: avstring.c:37
ff_h264_handle_frag_packet
int ff_h264_handle_frag_packet(AVPacket *pkt, const uint8_t *buf, int len, int start_bit, const uint8_t *nal_header, int nal_header_len)
Definition: rtpdec_h264.c:264
ff_hevc_dynamic_handler
const RTPDynamicProtocolHandler ff_hevc_dynamic_handler
Definition: rtpdec_hevc.c:341
AV_CODEC_ID_HEVC
@ AV_CODEC_ID_HEVC
Definition: codec_id.h:226
value
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default value
Definition: writing_filters.txt:86
PayloadContext::using_donl_field
int using_donl_field
Definition: rtpdec_hevc.c:39
len
int len
Definition: vorbis_enc_data.h:426
RTP_HEVC_DOND_FIELD_SIZE
#define RTP_HEVC_DOND_FIELD_SIZE
Definition: rtpdec_hevc.c:33
hevc_sdp_parse_fmtp_config
static av_cold int hevc_sdp_parse_fmtp_config(AVFormatContext *s, AVStream *stream, PayloadContext *hevc_data, const char *attr, const char *value)
Definition: rtpdec_hevc.c:47
ret
ret
Definition: filter_design.txt:187
AVStream
Stream structure.
Definition: avformat.h:948
pos
unsigned int pos
Definition: spdifenc.c:412
avformat.h
PayloadContext::vps
uint8_t * vps
Definition: rtpdec_hevc.c:41
AVStream::index
int index
stream index in AVFormatContext
Definition: avformat.h:956
AVPacket::stream_index
int stream_index
Definition: packet.h:376
parse_packet
static int parse_packet(AVFormatContext *s, AVPacket *pkt, int stream_index, int flush)
Parse a packet, add all split parts to parse_queue.
Definition: demux.c:1119
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
ff_h264_handle_aggregated_packet
int ff_h264_handle_aggregated_packet(AVFormatContext *ctx, PayloadContext *data, AVPacket *pkt, const uint8_t *buf, int len, int start_skip, int *nal_counters, int nal_mask)
Definition: rtpdec_h264.c:206
PayloadContext::sei_size
int sei_size
Definition: rtpdec_hevc.c:42
AVPacket
This structure stores compressed data.
Definition: packet.h:351
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:34
AVSTREAM_PARSE_FULL
@ AVSTREAM_PARSE_FULL
full parsing and repack
Definition: avformat.h:798
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:561
av_log
#define av_log(a,...)
Definition: tableprint_vlc.h:27
AVERROR_INVALIDDATA
#define AVERROR_INVALIDDATA
Invalid data found when processing input.
Definition: error.h:61
avstring.h
PayloadContext
RTP/JPEG specific private data.
Definition: rdt.c:83
PayloadContext::pps_size
int pps_size
Definition: rtpdec_hevc.c:42
ff_h264_parse_sprop_parameter_sets
int ff_h264_parse_sprop_parameter_sets(AVFormatContext *s, uint8_t **data_ptr, int *size_ptr, const char *value)
Definition: rtpdec_h264.c:96
line
The official guide to swscale for confused that consecutive non overlapping rectangles of slice_bottom special converter These generally are unscaled converters of common like for each output line the vertical scaler pulls lines from a ring buffer When the ring buffer does not contain the wanted line
Definition: swscale.txt:40
RTPDynamicProtocolHandler
Definition: rtpdec.h:116
ff_alloc_extradata
int ff_alloc_extradata(AVCodecParameters *par, int size)
Allocate extradata with additional AV_INPUT_BUFFER_PADDING_SIZE at end which is always set to 0.
Definition: utils.c:238