FFmpeg
framesync.h
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2013 Nicolas George
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public License
8  * as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14  * GNU Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public License
17  * along with FFmpeg; if not, write to the Free Software Foundation, Inc.,
18  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #ifndef AVFILTER_FRAMESYNC_H
22 #define AVFILTER_FRAMESYNC_H
23 
24 #include "bufferqueue.h"
25 
26 enum EOFAction {
30 };
31 
32 /*
33  * TODO
34  * Export convenient options.
35  */
36 
37 /**
38  * This API is intended as a helper for filters that have several video
39  * input and need to combine them somehow. If the inputs have different or
40  * variable frame rate, getting the input frames to match requires a rather
41  * complex logic and a few user-tunable options.
42  *
43  * In this API, when a set of synchronized input frames is ready to be
44  * procesed is called a frame event. Frame event can be generated in
45  * response to input frames on any or all inputs and the handling of
46  * situations where some stream extend beyond the beginning or the end of
47  * others can be configured.
48  *
49  * The basic working of this API is the following: set the on_event
50  * callback, then call ff_framesync_activate() from the filter's activate
51  * callback.
52  */
53 
54 /**
55  * Stream extrapolation mode
56  *
57  * Describe how the frames of a stream are extrapolated before the first one
58  * and after EOF to keep sync with possibly longer other streams.
59  */
61 
62  /**
63  * Completely stop all streams with this one.
64  */
66 
67  /**
68  * Ignore this stream and continue processing the other ones.
69  */
71 
72  /**
73  * Extend the frame to infinity.
74  */
76 };
77 
78 /**
79  * Timestamp syncronization mode
80  *
81  * Describe how the frames of a stream are syncronized based on timestamp
82  * distance.
83  */
85 
86  /**
87  * Sync to frames from secondary input with the nearest, lower or equal
88  * timestamp to the frame event one.
89  */
91 
92  /**
93  * Sync to frames from secondary input with the absolute nearest timestamp
94  * to the frame event one.
95  */
97 };
98 
99 /**
100  * Input stream structure
101  */
102 typedef struct FFFrameSyncIn {
103 
104  /**
105  * Extrapolation mode for timestamps before the first frame
106  */
108 
109  /**
110  * Extrapolation mode for timestamps after the last frame
111  */
113 
114  /**
115  * Time base for the incoming frames
116  */
118 
119  /**
120  * Current frame, may be NULL before the first one or after EOF
121  */
123 
124  /**
125  * Next frame, for internal use
126  */
128 
129  /**
130  * PTS of the current frame
131  */
132  int64_t pts;
133 
134  /**
135  * PTS of the next frame, for internal use
136  */
137  int64_t pts_next;
138 
139  /**
140  * Boolean flagging the next frame, for internal use
141  */
142  uint8_t have_next;
143 
144  /**
145  * State: before first, in stream or after EOF, for internal use
146  */
147  uint8_t state;
148 
149  /**
150  * Synchronization level: frames on input at the highest sync level will
151  * generate output frame events.
152  *
153  * For example, if inputs #0 and #1 have sync level 2 and input #2 has
154  * sync level 1, then a frame on either input #0 or #1 will generate a
155  * frame event, but not a frame on input #2 until both inputs #0 and #1
156  * have reached EOF.
157  *
158  * If sync is 0, no frame event will be generated.
159  */
160  unsigned sync;
161 
163 } FFFrameSyncIn;
164 
165 /**
166  * Frame sync structure.
167  */
168 typedef struct FFFrameSync {
169  const AVClass *class;
170 
171  /**
172  * Parent filter context.
173  */
175 
176  /**
177  * Number of input streams
178  */
179  unsigned nb_in;
180 
181  /**
182  * Time base for the output events
183  */
185 
186  /**
187  * Timestamp of the current event
188  */
189  int64_t pts;
190 
191  /**
192  * Callback called when a frame event is ready
193  */
194  int (*on_event)(struct FFFrameSync *fs);
195 
196  /**
197  * Opaque pointer, not used by the API
198  */
199  void *opaque;
200 
201  /**
202  * Index of the input that requires a request
203  */
204  unsigned in_request;
205 
206  /**
207  * Synchronization level: only inputs with the same sync level are sync
208  * sources.
209  */
210  unsigned sync_level;
211 
212  /**
213  * Flag indicating that a frame event is ready
214  */
215  uint8_t frame_ready;
216 
217  /**
218  * Flag indicating that output has reached EOF.
219  */
220  uint8_t eof;
221 
222  /**
223  * Pointer to array of inputs.
224  */
226 
231 
232 } FFFrameSync;
233 
234 /**
235  * Pre-initialize a frame sync structure.
236  *
237  * It sets the class pointer and inits the options to their default values.
238  * The entire structure is expected to be already set to 0.
239  * This step is optional, but necessary to use the options.
240  */
242 
243 /**
244  * Initialize a frame sync structure.
245  *
246  * The entire structure is expected to be already set to 0 or preinited.
247  *
248  * @param fs frame sync structure to initialize
249  * @param parent parent AVFilterContext object
250  * @param nb_in number of inputs
251  * @return >= 0 for success or a negative error code
252  */
254 
255 /**
256  * Configure a frame sync structure.
257  *
258  * Must be called after all options are set but before all use.
259  *
260  * @return >= 0 for success or a negative error code
261  */
263 
264 /**
265  * Free all memory currently allocated.
266  */
268 
269 /**
270  * Get the current frame in an input.
271  *
272  * @param fs frame sync structure
273  * @param in index of the input
274  * @param rframe used to return the current frame (or NULL)
275  * @param get if not zero, the calling code needs to get ownership of
276  * the returned frame; the current frame will either be
277  * duplicated or removed from the framesync structure
278  */
279 int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe,
280  unsigned get);
281 
282 /**
283  * Examine the frames in the filter's input and try to produce output.
284  *
285  * This function can be the complete implementation of the activate
286  * method of a filter using framesync.
287  */
289 
290 /**
291  * Initialize a frame sync structure for dualinput.
292  *
293  * Compared to generic framesync, dualinput assumes the first input is the
294  * main one and the filtering is performed on it. The first input will be
295  * the only one with sync set and generic timeline support will just pass it
296  * unchanged when disabled.
297  *
298  * Equivalent to ff_framesync_init(fs, parent, 2) then setting the time
299  * base, sync and ext modes on the inputs.
300  */
302 
303 /**
304  * @param f0 used to return the main frame
305  * @param f1 used to return the second frame, or NULL if disabled
306  * @return >=0 for success or AVERROR code
307  * @note The frame returned in f0 belongs to the caller (get = 1 in
308  * ff_framesync_get_frame()) while the frame returned in f1 is still owned
309  * by the framesync structure.
310  */
312 
313 /**
314  * Same as ff_framesync_dualinput_get(), but make sure that f0 is writable.
315  */
317 
318 const AVClass *ff_framesync_child_class_iterate(void **iter);
319 
320 #define FRAMESYNC_DEFINE_PURE_CLASS(name, desc, func_prefix, options) \
321 static const AVClass name##_class = { \
322  .class_name = desc, \
323  .item_name = av_default_item_name, \
324  .option = options, \
325  .version = LIBAVUTIL_VERSION_INT, \
326  .category = AV_CLASS_CATEGORY_FILTER, \
327  .child_class_iterate = ff_framesync_child_class_iterate, \
328  .child_next = func_prefix##_child_next, \
329 }
330 
331 /* A filter that uses the *_child_next-function from this macro
332  * is required to initialize the FFFrameSync structure in AVFilter.preinit
333  * via the *_framesync_preinit function defined alongside it. */
334 #define FRAMESYNC_AUXILIARY_FUNCS(func_prefix, context, field) \
335 static int func_prefix##_framesync_preinit(AVFilterContext *ctx) \
336 { \
337  context *s = ctx->priv; \
338  ff_framesync_preinit(&s->field); \
339  return 0; \
340 } \
341 static void *func_prefix##_child_next(void *obj, void *prev) \
342 { \
343  context *s = obj; \
344  return prev ? NULL : &s->field; \
345 }
346 
347 #define FRAMESYNC_DEFINE_CLASS_EXT(name, context, field, options) \
348 FRAMESYNC_AUXILIARY_FUNCS(name, context, field) \
349 FRAMESYNC_DEFINE_PURE_CLASS(name, #name, name, options)
350 
351 #define FRAMESYNC_DEFINE_CLASS(name, context, field) \
352 FRAMESYNC_DEFINE_CLASS_EXT(name, context, field, name##_options)
353 
354 #endif /* AVFILTER_FRAMESYNC_H */
FFFrameSyncIn::time_base
AVRational time_base
Time base for the incoming frames.
Definition: framesync.h:117
FFFrameSync::opt_repeatlast
int opt_repeatlast
Definition: framesync.h:227
FFFrameSyncIn::ts_mode
enum FFFrameTSSyncMode ts_mode
Definition: framesync.h:162
FFFrameSync::in_request
unsigned in_request
Index of the input that requires a request.
Definition: framesync.h:204
FFFrameSync::time_base
AVRational time_base
Time base for the output events.
Definition: framesync.h:184
FFFrameSyncIn::pts
int64_t pts
PTS of the current frame.
Definition: framesync.h:132
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:325
FFFrameSync::opt_shortest
int opt_shortest
Definition: framesync.h:228
FFFrameSync::sync_level
unsigned sync_level
Synchronization level: only inputs with the same sync level are sync sources.
Definition: framesync.h:210
FFFrameSync::frame_ready
uint8_t frame_ready
Flag indicating that a frame event is ready.
Definition: framesync.h:215
EOF_ACTION_ENDALL
@ EOF_ACTION_ENDALL
Definition: framesync.h:28
FFFrameTSSyncMode
FFFrameTSSyncMode
Timestamp syncronization mode.
Definition: framesync.h:84
FFFrameSync
Frame sync structure.
Definition: framesync.h:168
EXT_INFINITY
@ EXT_INFINITY
Extend the frame to infinity.
Definition: framesync.h:75
ff_framesync_init
int ff_framesync_init(FFFrameSync *fs, AVFilterContext *parent, unsigned nb_in)
Initialize a frame sync structure.
Definition: framesync.c:86
FFFrameSync::parent
AVFilterContext * parent
Parent filter context.
Definition: framesync.h:174
ff_framesync_get_frame
int ff_framesync_get_frame(FFFrameSync *fs, unsigned in, AVFrame **rframe, unsigned get)
Get the current frame in an input.
Definition: framesync.c:267
FFFrameSyncIn::state
uint8_t state
State: before first, in stream or after EOF, for internal use.
Definition: framesync.h:147
EXT_STOP
@ EXT_STOP
Completely stop all streams with this one.
Definition: framesync.h:65
FFFrameSyncExtMode
FFFrameSyncExtMode
This API is intended as a helper for filters that have several video input and need to combine them s...
Definition: framesync.h:60
FFFrameSync::on_event
int(* on_event)(struct FFFrameSync *fs)
Callback called when a frame event is ready.
Definition: framesync.h:194
FFFrameSyncIn::frame
AVFrame * frame
Current frame, may be NULL before the first one or after EOF.
Definition: framesync.h:122
FFFrameSyncIn
Input stream structure.
Definition: framesync.h:102
EXT_NULL
@ EXT_NULL
Ignore this stream and continue processing the other ones.
Definition: framesync.h:70
EOFAction
EOFAction
Definition: framesync.h:26
FFFrameSyncIn::sync
unsigned sync
Synchronization level: frames on input at the highest sync level will generate output frame events.
Definition: framesync.h:160
FFFrameSyncIn::pts_next
int64_t pts_next
PTS of the next frame, for internal use.
Definition: framesync.h:137
EOF_ACTION_PASS
@ EOF_ACTION_PASS
Definition: framesync.h:29
FFFrameSync::in
FFFrameSyncIn * in
Pointer to array of inputs.
Definition: framesync.h:225
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
TS_DEFAULT
@ TS_DEFAULT
Sync to frames from secondary input with the nearest, lower or equal timestamp to the frame event one...
Definition: framesync.h:90
fs
#define fs(width, name, subs,...)
Definition: cbs_vp9.c:258
AVRational
Rational number (pair of numerator and denominator).
Definition: rational.h:58
get
static void get(const uint8_t *pixels, int stride, int16_t *block)
Definition: proresenc_anatoliy.c:307
ff_framesync_init_dualinput
int ff_framesync_init_dualinput(FFFrameSync *fs, AVFilterContext *parent)
Initialize a frame sync structure for dualinput.
Definition: framesync.c:372
ff_framesync_dualinput_get_writable
int ff_framesync_dualinput_get_writable(FFFrameSync *fs, AVFrame **f0, AVFrame **f1)
Same as ff_framesync_dualinput_get(), but make sure that f0 is writable.
Definition: framesync.c:410
FFFrameSync::pts
int64_t pts
Timestamp of the current event.
Definition: framesync.h:189
bufferqueue.h
FFFrameSync::opaque
void * opaque
Opaque pointer, not used by the API.
Definition: framesync.h:199
FFFrameSync::opt_ts_sync_mode
int opt_ts_sync_mode
Definition: framesync.h:230
FFFrameSync::eof
uint8_t eof
Flag indicating that output has reached EOF.
Definition: framesync.h:220
FFFrameSyncIn::have_next
uint8_t have_next
Boolean flagging the next frame, for internal use.
Definition: framesync.h:142
FFFrameSyncIn::frame_next
AVFrame * frame_next
Next frame, for internal use.
Definition: framesync.h:127
EOF_ACTION_REPEAT
@ EOF_ACTION_REPEAT
Definition: framesync.h:27
FFFrameSync::opt_eof_action
int opt_eof_action
Definition: framesync.h:229
FFFrameSyncIn::before
enum FFFrameSyncExtMode before
Extrapolation mode for timestamps before the first frame.
Definition: framesync.h:107
ff_framesync_dualinput_get
int ff_framesync_dualinput_get(FFFrameSync *fs, AVFrame **f0, AVFrame **f1)
Definition: framesync.c:390
ff_framesync_child_class_iterate
const AVClass * ff_framesync_child_class_iterate(void **iter)
Definition: framesync.c:63
TS_NEAREST
@ TS_NEAREST
Sync to frames from secondary input with the absolute nearest timestamp to the frame event one.
Definition: framesync.h:96
ff_framesync_preinit
void ff_framesync_preinit(FFFrameSync *fs)
Pre-initialize a frame sync structure.
Definition: framesync.c:78
AVFilterContext
An instance of a filter.
Definition: avfilter.h:415
ff_framesync_configure
int ff_framesync_configure(FFFrameSync *fs)
Configure a frame sync structure.
Definition: framesync.c:134
ff_framesync_activate
int ff_framesync_activate(FFFrameSync *fs)
Examine the frames in the filter's input and try to produce output.
Definition: framesync.c:355
ff_framesync_uninit
void ff_framesync_uninit(FFFrameSync *fs)
Free all memory currently allocated.
Definition: framesync.c:304
FFFrameSyncIn::after
enum FFFrameSyncExtMode after
Extrapolation mode for timestamps after the last frame.
Definition: framesync.h:112
FFFrameSync::nb_in
unsigned nb_in
Number of input streams.
Definition: framesync.h:179
int
int
Definition: ffmpeg_filter.c:156