FFmpeg
vf_codecview.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
3  * Copyright (c) 2014 Clément Bœsch <u pkh me>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * Codec debug viewer filter.
25  *
26  * All the MV drawing code from Michael Niedermayer is extracted from
27  * libavcodec/mpegvideo.c.
28  *
29  * TODO: segmentation
30  */
31 
32 #include "libavutil/imgutils.h"
34 #include "libavutil/opt.h"
35 #include "avfilter.h"
36 #include "internal.h"
37 
38 #define MV_P_FOR (1<<0)
39 #define MV_B_FOR (1<<1)
40 #define MV_B_BACK (1<<2)
41 #define MV_TYPE_FOR (1<<0)
42 #define MV_TYPE_BACK (1<<1)
43 #define FRAME_TYPE_I (1<<0)
44 #define FRAME_TYPE_P (1<<1)
45 #define FRAME_TYPE_B (1<<2)
46 
47 typedef struct CodecViewContext {
48  const AVClass *class;
49  unsigned mv;
50  unsigned frame_type;
51  unsigned mv_type;
52  int hsub, vsub;
53  int qp;
55 
56 #define OFFSET(x) offsetof(CodecViewContext, x)
57 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
58 #define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, FLAGS, unit }
59 
60 static const AVOption codecview_options[] = {
61  { "mv", "set motion vectors to visualize", OFFSET(mv), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv" },
62  CONST("pf", "forward predicted MVs of P-frames", MV_P_FOR, "mv"),
63  CONST("bf", "forward predicted MVs of B-frames", MV_B_FOR, "mv"),
64  CONST("bb", "backward predicted MVs of B-frames", MV_B_BACK, "mv"),
65  { "qp", NULL, OFFSET(qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
66  { "mv_type", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
67  { "mvt", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
68  CONST("fp", "forward predicted MVs", MV_TYPE_FOR, "mv_type"),
69  CONST("bp", "backward predicted MVs", MV_TYPE_BACK, "mv_type"),
70  { "frame_type", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
71  { "ft", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
72  CONST("if", "I-frames", FRAME_TYPE_I, "frame_type"),
73  CONST("pf", "P-frames", FRAME_TYPE_P, "frame_type"),
74  CONST("bf", "B-frames", FRAME_TYPE_B, "frame_type"),
75  { NULL }
76 };
77 
78 AVFILTER_DEFINE_CLASS(codecview);
79 
81 {
82  // TODO: we can probably add way more pixel formats without any other
83  // changes; anything with 8-bit luma in first plane should be working
86  if (!fmts_list)
87  return AVERROR(ENOMEM);
88  return ff_set_common_formats(ctx, fmts_list);
89 }
90 
91 static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
92 {
93  if(*sx > *ex)
94  return clip_line(ex, ey, sx, sy, maxx);
95 
96  if (*sx < 0) {
97  if (*ex < 0)
98  return 1;
99  *sy = *ey + (*sy - *ey) * (int64_t)*ex / (*ex - *sx);
100  *sx = 0;
101  }
102 
103  if (*ex > maxx) {
104  if (*sx > maxx)
105  return 1;
106  *ey = *sy + (*ey - *sy) * (int64_t)(maxx - *sx) / (*ex - *sx);
107  *ex = maxx;
108  }
109  return 0;
110 }
111 
112 /**
113  * Draw a line from (ex, ey) -> (sx, sy).
114  * @param w width of the image
115  * @param h height of the image
116  * @param stride stride/linesize of the image
117  * @param color color of the arrow
118  */
119 static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey,
120  int w, int h, int stride, int color)
121 {
122  int x, y, fr, f;
123 
124  if (clip_line(&sx, &sy, &ex, &ey, w - 1))
125  return;
126  if (clip_line(&sy, &sx, &ey, &ex, h - 1))
127  return;
128 
129  sx = av_clip(sx, 0, w - 1);
130  sy = av_clip(sy, 0, h - 1);
131  ex = av_clip(ex, 0, w - 1);
132  ey = av_clip(ey, 0, h - 1);
133 
134  buf[sy * stride + sx] += color;
135 
136  if (FFABS(ex - sx) > FFABS(ey - sy)) {
137  if (sx > ex) {
138  FFSWAP(int, sx, ex);
139  FFSWAP(int, sy, ey);
140  }
141  buf += sx + sy * stride;
142  ex -= sx;
143  f = ((ey - sy) << 16) / ex;
144  for (x = 0; x <= ex; x++) {
145  y = (x * f) >> 16;
146  fr = (x * f) & 0xFFFF;
147  buf[ y * stride + x] += (color * (0x10000 - fr)) >> 16;
148  if(fr) buf[(y + 1) * stride + x] += (color * fr ) >> 16;
149  }
150  } else {
151  if (sy > ey) {
152  FFSWAP(int, sx, ex);
153  FFSWAP(int, sy, ey);
154  }
155  buf += sx + sy * stride;
156  ey -= sy;
157  if (ey)
158  f = ((ex - sx) << 16) / ey;
159  else
160  f = 0;
161  for(y= 0; y <= ey; y++){
162  x = (y*f) >> 16;
163  fr = (y*f) & 0xFFFF;
164  buf[y * stride + x ] += (color * (0x10000 - fr)) >> 16;
165  if(fr) buf[y * stride + x + 1] += (color * fr ) >> 16;
166  }
167  }
168 }
169 
170 /**
171  * Draw an arrow from (ex, ey) -> (sx, sy).
172  * @param w width of the image
173  * @param h height of the image
174  * @param stride stride/linesize of the image
175  * @param color color of the arrow
176  */
177 static void draw_arrow(uint8_t *buf, int sx, int sy, int ex,
178  int ey, int w, int h, int stride, int color, int tail, int direction)
179 {
180  int dx,dy;
181 
182  if (direction) {
183  FFSWAP(int, sx, ex);
184  FFSWAP(int, sy, ey);
185  }
186 
187  sx = av_clip(sx, -100, w + 100);
188  sy = av_clip(sy, -100, h + 100);
189  ex = av_clip(ex, -100, w + 100);
190  ey = av_clip(ey, -100, h + 100);
191 
192  dx = ex - sx;
193  dy = ey - sy;
194 
195  if (dx * dx + dy * dy > 3 * 3) {
196  int rx = dx + dy;
197  int ry = -dx + dy;
198  int length = sqrt((rx * rx + ry * ry) << 8);
199 
200  // FIXME subpixel accuracy
201  rx = ROUNDED_DIV(rx * 3 << 4, length);
202  ry = ROUNDED_DIV(ry * 3 << 4, length);
203 
204  if (tail) {
205  rx = -rx;
206  ry = -ry;
207  }
208 
209  draw_line(buf, sx, sy, sx + rx, sy + ry, w, h, stride, color);
210  draw_line(buf, sx, sy, sx - ry, sy + rx, w, h, stride, color);
211  }
212  draw_line(buf, sx, sy, ex, ey, w, h, stride, color);
213 }
214 
216 {
217  AVFilterContext *ctx = inlink->dst;
218  CodecViewContext *s = ctx->priv;
219  AVFilterLink *outlink = ctx->outputs[0];
220 
221  if (s->qp) {
222  int qstride, qp_type;
223  int8_t *qp_table = av_frame_get_qp_table(frame, &qstride, &qp_type);
224 
225  if (qp_table) {
226  int x, y;
227  const int w = AV_CEIL_RSHIFT(frame->width, s->hsub);
228  const int h = AV_CEIL_RSHIFT(frame->height, s->vsub);
229  uint8_t *pu = frame->data[1];
230  uint8_t *pv = frame->data[2];
231  const int lzu = frame->linesize[1];
232  const int lzv = frame->linesize[2];
233 
234  for (y = 0; y < h; y++) {
235  for (x = 0; x < w; x++) {
236  const int qp = ff_norm_qscale(qp_table[(y >> 3) * qstride + (x >> 3)], qp_type) * 128/31;
237  pu[x] = pv[x] = qp;
238  }
239  pu += lzu;
240  pv += lzv;
241  }
242  }
243  }
244 
245  if (s->mv || s->mv_type) {
247  if (sd) {
248  int i;
249  const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
250  const int is_iframe = (s->frame_type & FRAME_TYPE_I) && frame->pict_type == AV_PICTURE_TYPE_I;
251  const int is_pframe = (s->frame_type & FRAME_TYPE_P) && frame->pict_type == AV_PICTURE_TYPE_P;
252  const int is_bframe = (s->frame_type & FRAME_TYPE_B) && frame->pict_type == AV_PICTURE_TYPE_B;
253 
254  for (i = 0; i < sd->size / sizeof(*mvs); i++) {
255  const AVMotionVector *mv = &mvs[i];
256  const int direction = mv->source > 0;
257 
258  if (s->mv_type) {
259  const int is_fp = direction == 0 && (s->mv_type & MV_TYPE_FOR);
260  const int is_bp = direction == 1 && (s->mv_type & MV_TYPE_BACK);
261 
262  if ((!s->frame_type && (is_fp || is_bp)) ||
263  is_iframe && is_fp || is_iframe && is_bp ||
264  is_pframe && is_fp ||
265  is_bframe && is_fp || is_bframe && is_bp)
266  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
267  frame->width, frame->height, frame->linesize[0],
268  100, 0, direction);
269  } else if (s->mv)
270  if ((direction == 0 && (s->mv & MV_P_FOR) && frame->pict_type == AV_PICTURE_TYPE_P) ||
271  (direction == 0 && (s->mv & MV_B_FOR) && frame->pict_type == AV_PICTURE_TYPE_B) ||
272  (direction == 1 && (s->mv & MV_B_BACK) && frame->pict_type == AV_PICTURE_TYPE_B))
273  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
274  frame->width, frame->height, frame->linesize[0],
275  100, 0, direction);
276  }
277  }
278  }
279 
280  return ff_filter_frame(outlink, frame);
281 }
282 
284 {
285  AVFilterContext *ctx = inlink->dst;
286  CodecViewContext *s = ctx->priv;
288 
289  s->hsub = desc->log2_chroma_w;
290  s->vsub = desc->log2_chroma_h;
291  return 0;
292 }
293 
294 static const AVFilterPad codecview_inputs[] = {
295  {
296  .name = "default",
297  .type = AVMEDIA_TYPE_VIDEO,
298  .filter_frame = filter_frame,
299  .config_props = config_input,
300  .needs_writable = 1,
301  },
302  { NULL }
303 };
304 
305 static const AVFilterPad codecview_outputs[] = {
306  {
307  .name = "default",
308  .type = AVMEDIA_TYPE_VIDEO,
309  },
310  { NULL }
311 };
312 
314  .name = "codecview",
315  .description = NULL_IF_CONFIG_SMALL("Visualize information about some codecs."),
316  .priv_size = sizeof(CodecViewContext),
320  .priv_class = &codecview_class,
322 };
stride
int stride
Definition: mace.c:144
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
MV_B_FOR
#define MV_B_FOR
Definition: vf_codecview.c:39
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
draw_line
static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color)
Draw a line from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:119
ff_make_format_list
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:300
FFSWAP
#define FFSWAP(type, a, b)
Definition: common.h:99
color
Definition: vf_paletteuse.c:582
av_frame_get_side_data
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:739
draw_arrow
static void draw_arrow(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color, int tail, int direction)
Draw an arrow from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:177
AVMotionVector
Definition: motion_vector.h:24
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1075
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2549
mv
static const int8_t mv[256][2]
Definition: 4xm.c:77
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:300
w
uint8_t w
Definition: llviddspenc.c:38
AVOption
AVOption.
Definition: opt.h:246
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:148
MV_P_FOR
#define MV_P_FOR
Definition: vf_codecview.c:38
query_formats
static int query_formats(AVFilterContext *ctx)
Definition: vf_codecview.c:80
CONST
#define CONST(name, help, val, unit)
Definition: vf_codecview.c:58
OFFSET
#define OFFSET(x)
Definition: vf_codecview.c:56
AVFilterFormats
A list of supported formats for one end of a filter link.
Definition: formats.h:64
av_frame_get_qp_table
int8_t * av_frame_get_qp_table(AVFrame *f, int *stride, int *type)
Definition: frame.c:91
CodecViewContext::hsub
int hsub
Definition: vf_codecview.c:52
clip_line
static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
Definition: vf_codecview.c:91
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:54
ff_set_common_formats
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:605
CodecViewContext::mv_type
unsigned mv_type
Definition: vf_codecview.c:51
motion_vector.h
s
#define s(width, name)
Definition: cbs_vp9.c:257
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:275
ctx
AVFormatContext * ctx
Definition: movenc.c:48
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
f
#define f(width, name)
Definition: cbs_vp9.c:255
FFABS
#define FFABS(a)
Absolute value, Note, INT_MIN / INT64_MIN result in undefined behavior as they are not representable ...
Definition: common.h:72
codecview_inputs
static const AVFilterPad codecview_inputs[]
Definition: vf_codecview.c:294
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:67
NULL
#define NULL
Definition: coverity.c:32
ROUNDED_DIV
#define ROUNDED_DIV(a, b)
Definition: common.h:56
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
desc
const char * desc
Definition: nvenc.c:79
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:188
codecview_options
static const AVOption codecview_options[]
Definition: vf_codecview.c:60
MV_B_BACK
#define MV_B_BACK
Definition: vf_codecview.c:40
color
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
AVFrameSideData::data
uint8_t * data
Definition: frame.h:208
CodecViewContext
Definition: vf_codecview.c:47
ff_norm_qscale
static int ff_norm_qscale(int qscale, int type)
Normalize the qscale factor FIXME the H264 qscale is a log based scale, mpeg1/2 is not,...
Definition: internal.h:377
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(codecview)
FRAME_TYPE_I
#define FRAME_TYPE_I
Definition: vf_codecview.c:43
codecview_outputs
static const AVFilterPad codecview_outputs[]
Definition: vf_codecview.c:305
internal.h
AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
#define AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
Some filters support a generic "enable" expression option that can be used to enable or disable a fil...
Definition: avfilter.h:125
CodecViewContext::vsub
int vsub
Definition: vf_codecview.c:52
i
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:269
frame_type
frame_type
Definition: jpeg2000_parser.c:31
uint8_t
uint8_t
Definition: audio_convert.c:194
FRAME_TYPE_B
#define FRAME_TYPE_B
Definition: vf_codecview.c:45
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:60
pv
#define pv
Definition: regdef.h:60
AVFilter
Filter definition.
Definition: avfilter.h:144
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
filter_frame
static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_codecview.c:215
AV_PICTURE_TYPE_B
@ AV_PICTURE_TYPE_B
Bi-dir predicted.
Definition: avutil.h:276
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
avfilter.h
FLAGS
#define FLAGS
Definition: vf_codecview.c:57
AVFilterContext
An instance of a filter.
Definition: avfilter.h:338
AV_PICTURE_TYPE_P
@ AV_PICTURE_TYPE_P
Predicted.
Definition: avutil.h:275
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
MV_TYPE_FOR
#define MV_TYPE_FOR
Definition: vf_codecview.c:41
MV_TYPE_BACK
#define MV_TYPE_BACK
Definition: vf_codecview.c:42
AVFrameSideData
Structure to hold side data for an AVFrame.
Definition: frame.h:206
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:81
CodecViewContext::qp
int qp
Definition: vf_codecview.c:53
AV_OPT_TYPE_BOOL
@ AV_OPT_TYPE_BOOL
Definition: opt.h:240
AVFrameSideData::size
int size
Definition: frame.h:209
CodecViewContext::mv
unsigned mv
Definition: vf_codecview.c:49
AV_OPT_TYPE_FLAGS
@ AV_OPT_TYPE_FLAGS
Definition: opt.h:222
imgutils.h
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:565
AV_FRAME_DATA_MOTION_VECTORS
@ AV_FRAME_DATA_MOTION_VECTORS
Motion vectors exported by some codecs (on demand through the export_mvs flag set in the libavcodec A...
Definition: frame.h:96
config_input
static int config_input(AVFilterLink *inlink)
Definition: vf_codecview.c:283
h
h
Definition: vp9dsp_template.c:2038
CodecViewContext::frame_type
unsigned frame_type
Definition: vf_codecview.c:50
FRAME_TYPE_P
#define FRAME_TYPE_P
Definition: vf_codecview.c:44
ff_vf_codecview
AVFilter ff_vf_codecview
Definition: vf_codecview.c:313