FFmpeg
 All Data Structures Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Groups Pages
framesync.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2013 Nicolas George
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public License
8  * as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #ifndef AVFILTER_FRAMESYNC_H
22 #define AVFILTER_FRAMESYNC_H
23 
24 #include "bufferqueue.h"
25 
26 /*
27  * TODO
28  * Callback-based API similar to dualinput.
29  * Export convenient options.
30  */
31 
32 /**
33  * This API is intended as a helper for filters that have several video
34  * input and need to combine them somehow. If the inputs have different or
35  * variable frame rate, getting the input frames to match requires a rather
36  * complex logic and a few user-tunable options.
37  *
38  * In this API, when a set of synchronized input frames is ready to be
39  * procesed is called a frame event. Frame event can be generated in
40  * response to input frames on any or all inputs and the handling of
41  * situations where some stream extend beyond the beginning or the end of
42  * others can be configured.
43  *
44  * The basic working of this API is the following:
45  *
46  * - When a frame is available on any input, add it using
47  * ff_framesync_add_frame().
48  *
49  * - When a frame event is ready to be processed (i.e. after adding a frame
50  * or when requested on input):
51  * - call ff_framesync_next();
52  * - if fs->frame_ready is true, process the frames;
53  * - call ff_framesync_drop().
54  */
55 
56 /**
57  * Stream extrapolation mode
58  *
59  * Describe how the frames of a stream are extrapolated before the first one
60  * and after EOF to keep sync with possibly longer other streams.
61  */
63 
64  /**
65  * Completely stop all streams with this one.
66  */
68 
69  /**
70  * Ignore this stream and continue processing the other ones.
71  */
73 
74  /**
75  * Extend the frame to infinity.
76  */
78 };
79 
80 /**
81  * Input stream structure
82  */
83 typedef struct FFFrameSyncIn {
84 
85  /**
86  * Queue of incoming AVFrame, and NULL to mark EOF
87  */
88  struct FFBufQueue queue;
89 
90  /**
91  * Extrapolation mode for timestamps before the first frame
92  */
94 
95  /**
96  * Extrapolation mode for timestamps after the last frame
97  */
99 
100  /**
101  * Time base for the incoming frames
102  */
104 
105  /**
106  * Current frame, may be NULL before the first one or after EOF
107  */
109 
110  /**
111  * Next frame, for internal use
112  */
114 
115  /**
116  * PTS of the current frame
117  */
118  int64_t pts;
119 
120  /**
121  * PTS of the next frame, for internal use
122  */
123  int64_t pts_next;
124 
125  /**
126  * Boolean flagging the next frame, for internal use
127  */
129 
130  /**
131  * State: before first, in stream or after EOF, for internal use
132  */
134 
135  /**
136  * Synchronization level: frames on input at the highest sync level will
137  * generate output frame events.
138  *
139  * For example, if inputs #0 and #1 have sync level 2 and input #2 has
140  * sync level 1, then a frame on either input #0 or #1 will generate a
141  * frame event, but not a frame on input #2 until both inputs #0 and #1
142  * have reached EOF.
143  *
144  * If sync is 0, no frame event will be generated.
145  */
146  unsigned sync;
147 
148 } FFFrameSyncIn;
149 
150 /**
151  * Frame sync structure.
152  */
153 typedef struct FFFrameSync {
154  const AVClass *class;
155  void *parent;
156 
157  /**
158  * Number of input streams
159  */
160  unsigned nb_in;
161 
162  /**
163  * Time base for the output events
164  */
166 
167  /**
168  * Timestamp of the current event
169  */
170  int64_t pts;
171 
172  /**
173  * Callback called when a frame event is ready
174  */
175  int (*on_event)(struct FFFrameSync *fs);
176 
177  /**
178  * Opaque pointer, not used by the API
179  */
180  void *opaque;
181 
182  /**
183  * Index of the input that requires a request
184  */
185  unsigned in_request;
186 
187  /**
188  * Synchronization level: only inputs with the same sync level are sync
189  * sources.
190  */
191  unsigned sync_level;
192 
193  /**
194  * Flag indicating that a frame event is ready
195  */
197 
198  /**
199  * Flag indicating that output has reached EOF.
200  */
202 
203  /**
204  * Pointer to array of inputs.
205  */
207 
208 } FFFrameSync;
209 
210 /**
211  * Initialize a frame sync structure.
212  *
213  * The entire structure is expected to be already set to 0.
214  *
215  * @param fs frame sync structure to initialize
216  * @param parent parent object, used for logging
217  * @param nb_in number of inputs
218  * @return >= 0 for success or a negative error code
219  */
220 int ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in);
221 
222 /**
223  * Configure a frame sync structure.
224  *
225  * Must be called after all options are set but before all use.
226  *
227  * @return >= 0 for success or a negative error code
228  */
230 
231 /**
232  * Free all memory currently allocated.
233  */
235 
236 /**
237  * Add a frame to an input
238  *
239  * Typically called from the filter_frame() method.
240  *
241  * @param fs frame sync structure
242  * @param in index of the input
243  * @param frame input frame, or NULL for EOF
244  */
245 int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame);
246 
247 /**
248  * Prepare the next frame event.
249  *
250  * The status of the operation can be found in fs->frame_ready and fs->eof.
251  */
253 
254 /**
255  * Drop the current frame event.
256  */
258 
259 /**
260  * Get the current frame in an input.
261  *
262  * @param fs frame sync structure
263  * @param in index of the input
264  * @param rframe used to return the current frame (or NULL)
265  * @param get if not zero, the calling code needs to get ownership of
266  * the returned frame; the current frame will either be
267  * duplicated or removed from the framesync structure
268  */
269 int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe,
270  unsigned get);
271 
272 /**
273  * Process one or several frame using the on_event callback.
274  *
275  * @return number of frames processed or negative error code
276  */
277 int ff_framesync_process_frame(FFFrameSync *fs, unsigned all);
278 
279 
280 /**
281  * Accept a frame on a filter input.
282  *
283  * This function can be the complete implementation of all filter_frame
284  * methods of a filter using framesync.
285  */
287  AVFrame *in);
288 
289 /**
290  * Request a frame on the filter output.
291  *
292  * This function can be the complete implementation of all filter_frame
293  * methods of a filter using framesync if it has only one output.
294  */
296 
297 #endif /* AVFILTER_FRAMESYNC_H */
void ff_framesync_drop(FFFrameSync *fs)
Drop the current frame event.
Definition: framesync.c:224
This structure describes decoded (raw) audio or video data.
Definition: frame.h:171
int(* on_event)(struct FFFrameSync *fs)
Callback called when a frame event is ready.
Definition: framesync.h:175
int ff_framesync_request_frame(FFFrameSync *fs, AVFilterLink *outlink)
Request a frame on the filter output.
Definition: framesync.c:314
void ff_framesync_next(FFFrameSync *fs)
Prepare the next frame event.
Definition: framesync.c:212
int64_t pts
Timestamp of the current event.
Definition: framesync.h:170
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:93
Structure holding the queue.
Definition: bufferqueue.h:49
uint8_t have_next
Boolean flagging the next frame, for internal use.
Definition: framesync.h:128
uint8_t
void * parent
Definition: framesync.h:155
unsigned sync_level
Synchronization level: only inputs with the same sync level are sync sources.
Definition: framesync.h:191
FFFrameSyncIn * in
Pointer to array of inputs.
Definition: framesync.h:206
static AVFrame * frame
int ff_framesync_add_frame(FFFrameSync *fs, unsigned in, AVFrame *frame)
Add a frame to an input.
Definition: framesync.c:202
int ff_framesync_filter_frame(FFFrameSync *fs, AVFilterLink *inlink, AVFrame *in)
Accept a frame on a filter input.
Definition: framesync.c:300
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:98
Input stream structure.
Definition: framesync.h:83
AVFrame * frame_next
Next frame, for internal use.
Definition: framesync.h:113
Frame sync structure.
Definition: framesync.h:153
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:103
int ff_framesync_init(FFFrameSync *fs, void *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:49
uint8_t eof
Flag indicating that output has reached EOF.
Definition: framesync.h:201
unsigned in_request
Index of the input that requires a request.
Definition: framesync.h:185
FFFrameSyncExtMode
This API is intended as a helper for filters that have several video input and need to combine them s...
Definition: framesync.h:62
AVRational time_base
Time base for the output events.
Definition: framesync.h:165
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:229
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:77
struct FFBufQueue queue
Queue of incoming AVFrame, and NULL to mark EOF.
Definition: framesync.h:88
void * opaque
Opaque pointer, not used by the API.
Definition: framesync.h:180
Extend the frame to infinity.
Definition: framesync.h:77
uint8_t pi<< 24) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_U8, uint8_t,(*(constuint8_t *) pi-0x80)*(1.0f/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_U8, uint8_t,(*(constuint8_t *) pi-0x80)*(1.0/(1<< 7))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S16, int16_t,(*(constint16_t *) pi >>8)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S16, int16_t,*(constint16_t *) pi *(1.0f/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S16, int16_t,*(constint16_t *) pi *(1.0/(1<< 15))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_S32, int32_t,(*(constint32_t *) pi >>24)+0x80) CONV_FUNC_GROUP(AV_SAMPLE_FMT_FLT, float, AV_SAMPLE_FMT_S32, int32_t,*(constint32_t *) pi *(1.0f/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_DBL, double, AV_SAMPLE_FMT_S32, int32_t,*(constint32_t *) pi *(1.0/(1U<< 31))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_FLT, float, av_clip_uint8(lrintf(*(constfloat *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_FLT, float, av_clip_int16(lrintf(*(constfloat *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_FLT, float, av_clipl_int32(llrintf(*(constfloat *) pi *(1U<< 31)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_U8, uint8_t, AV_SAMPLE_FMT_DBL, double, av_clip_uint8(lrint(*(constdouble *) pi *(1<< 7))+0x80)) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S16, int16_t, AV_SAMPLE_FMT_DBL, double, av_clip_int16(lrint(*(constdouble *) pi *(1<< 15)))) CONV_FUNC_GROUP(AV_SAMPLE_FMT_S32, int32_t, AV_SAMPLE_FMT_DBL, double, av_clipl_int32(llrint(*(constdouble *) pi *(1U<< 31))))#defineSET_CONV_FUNC_GROUP(ofmt, ifmt) staticvoidset_generic_function(AudioConvert *ac){}voidff_audio_convert_free(AudioConvert **ac){if(!*ac) return;ff_dither_free(&(*ac) ->dc);av_freep(ac);}AudioConvert *ff_audio_convert_alloc(AVAudioResampleContext *avr, enumAVSampleFormatout_fmt, enumAVSampleFormatin_fmt, intchannels, intsample_rate, intapply_map){AudioConvert *ac;intin_planar, out_planar;ac=av_mallocz(sizeof(*ac));if(!ac) returnNULL;ac->avr=avr;ac->out_fmt=out_fmt;ac->in_fmt=in_fmt;ac->channels=channels;ac->apply_map=apply_map;if(avr->dither_method!=AV_RESAMPLE_DITHER_NONE &&av_get_packed_sample_fmt(out_fmt)==AV_SAMPLE_FMT_S16 &&av_get_bytes_per_sample(in_fmt)>2){ac->dc=ff_dither_alloc(avr, out_fmt, in_fmt, channels, sample_rate, apply_map);if(!ac->dc){av_free(ac);returnNULL;}returnac;}in_planar=ff_sample_fmt_is_planar(in_fmt, channels);out_planar=ff_sample_fmt_is_planar(out_fmt, channels);if(in_planar==out_planar){ac->func_type=CONV_FUNC_TYPE_FLAT;ac->planes=in_planar?ac->channels:1;}elseif(in_planar) ac->func_type=CONV_FUNC_TYPE_INTERLEAVE;elseac->func_type=CONV_FUNC_TYPE_DEINTERLEAVE;set_generic_function(ac);if(ARCH_AARCH64) ff_audio_convert_init_aarch64(ac);if(ARCH_ARM) ff_audio_convert_init_arm(ac);if(ARCH_X86) ff_audio_convert_init_x86(ac);returnac;}intff_audio_convert(AudioConvert *ac, AudioData *out, AudioData *in){intuse_generic=1;intlen=in->nb_samples;intp;if(ac->dc){av_log(ac->avr, AV_LOG_TRACE,"%dsamples-audio_convert:%sto%s(dithered)\n", len, av_get_sample_fmt_name(ac->in_fmt), av_get_sample_fmt_name(ac->out_fmt));returnff_convert_dither(ac-> in
uint8_t state
State: before first, in stream or after EOF, for internal use.
Definition: framesync.h:133
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events...
Definition: framesync.h:146
Describe the class of an AVClass context structure.
Definition: log.h:67
rational number numerator/denominator
Definition: rational.h:43
Ignore this stream and continue processing the other ones.
Definition: framesync.h:72
int ff_framesync_process_frame(FFFrameSync *fs, unsigned all)
Process one or several frame using the on_event callback.
Definition: framesync.c:279
unsigned nb_in
Number of input streams.
Definition: framesync.h:160
AVFrame * frame
Current frame, may be NULL before the first one or after EOF.
Definition: framesync.h:108
uint8_t frame_ready
Flag indicating that a frame event is ready.
Definition: framesync.h:196
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:266
int64_t pts
PTS of the current frame.
Definition: framesync.h:118
Completely stop all streams with this one.
Definition: framesync.h:67
int64_t pts_next
PTS of the next frame, for internal use.
Definition: framesync.h:123