FFmpeg
vf_codecview.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
3  * Copyright (c) 2014 Clément Bœsch <u pkh me>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * Codec debug viewer filter.
25  *
26  * All the MV drawing code from Michael Niedermayer is extracted from
27  * libavcodec/mpegvideo.c.
28  *
29  * TODO: segmentation
30  */
31 
32 #include "libavutil/imgutils.h"
34 #include "libavutil/opt.h"
35 #include "avfilter.h"
36 #include "qp_table.h"
37 #include "internal.h"
38 
39 #define MV_P_FOR (1<<0)
40 #define MV_B_FOR (1<<1)
41 #define MV_B_BACK (1<<2)
42 #define MV_TYPE_FOR (1<<0)
43 #define MV_TYPE_BACK (1<<1)
44 #define FRAME_TYPE_I (1<<0)
45 #define FRAME_TYPE_P (1<<1)
46 #define FRAME_TYPE_B (1<<2)
47 
48 typedef struct CodecViewContext {
49  const AVClass *class;
50  unsigned mv;
51  unsigned frame_type;
52  unsigned mv_type;
53  int hsub, vsub;
54  int qp;
56 
57 #define OFFSET(x) offsetof(CodecViewContext, x)
58 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
59 #define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, FLAGS, unit }
60 
61 static const AVOption codecview_options[] = {
62  { "mv", "set motion vectors to visualize", OFFSET(mv), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv" },
63  CONST("pf", "forward predicted MVs of P-frames", MV_P_FOR, "mv"),
64  CONST("bf", "forward predicted MVs of B-frames", MV_B_FOR, "mv"),
65  CONST("bb", "backward predicted MVs of B-frames", MV_B_BACK, "mv"),
66  { "qp", NULL, OFFSET(qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
67  { "mv_type", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
68  { "mvt", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
69  CONST("fp", "forward predicted MVs", MV_TYPE_FOR, "mv_type"),
70  CONST("bp", "backward predicted MVs", MV_TYPE_BACK, "mv_type"),
71  { "frame_type", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
72  { "ft", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
73  CONST("if", "I-frames", FRAME_TYPE_I, "frame_type"),
74  CONST("pf", "P-frames", FRAME_TYPE_P, "frame_type"),
75  CONST("bf", "B-frames", FRAME_TYPE_B, "frame_type"),
76  { NULL }
77 };
78 
79 AVFILTER_DEFINE_CLASS(codecview);
80 
82 {
83  // TODO: we can probably add way more pixel formats without any other
84  // changes; anything with 8-bit luma in first plane should be working
87  if (!fmts_list)
88  return AVERROR(ENOMEM);
89  return ff_set_common_formats(ctx, fmts_list);
90 }
91 
92 static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
93 {
94  if(*sx > *ex)
95  return clip_line(ex, ey, sx, sy, maxx);
96 
97  if (*sx < 0) {
98  if (*ex < 0)
99  return 1;
100  *sy = *ey + (*sy - *ey) * (int64_t)*ex / (*ex - *sx);
101  *sx = 0;
102  }
103 
104  if (*ex > maxx) {
105  if (*sx > maxx)
106  return 1;
107  *ey = *sy + (*ey - *sy) * (int64_t)(maxx - *sx) / (*ex - *sx);
108  *ex = maxx;
109  }
110  return 0;
111 }
112 
113 /**
114  * Draw a line from (ex, ey) -> (sx, sy).
115  * @param w width of the image
116  * @param h height of the image
117  * @param stride stride/linesize of the image
118  * @param color color of the arrow
119  */
120 static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey,
121  int w, int h, int stride, int color)
122 {
123  int x, y, fr, f;
124 
125  if (clip_line(&sx, &sy, &ex, &ey, w - 1))
126  return;
127  if (clip_line(&sy, &sx, &ey, &ex, h - 1))
128  return;
129 
130  sx = av_clip(sx, 0, w - 1);
131  sy = av_clip(sy, 0, h - 1);
132  ex = av_clip(ex, 0, w - 1);
133  ey = av_clip(ey, 0, h - 1);
134 
135  buf[sy * stride + sx] += color;
136 
137  if (FFABS(ex - sx) > FFABS(ey - sy)) {
138  if (sx > ex) {
139  FFSWAP(int, sx, ex);
140  FFSWAP(int, sy, ey);
141  }
142  buf += sx + sy * stride;
143  ex -= sx;
144  f = ((ey - sy) * (1 << 16)) / ex;
145  for (x = 0; x <= ex; x++) {
146  y = (x * f) >> 16;
147  fr = (x * f) & 0xFFFF;
148  buf[ y * stride + x] += (color * (0x10000 - fr)) >> 16;
149  if(fr) buf[(y + 1) * stride + x] += (color * fr ) >> 16;
150  }
151  } else {
152  if (sy > ey) {
153  FFSWAP(int, sx, ex);
154  FFSWAP(int, sy, ey);
155  }
156  buf += sx + sy * stride;
157  ey -= sy;
158  if (ey)
159  f = ((ex - sx) * (1 << 16)) / ey;
160  else
161  f = 0;
162  for(y= 0; y <= ey; y++){
163  x = (y*f) >> 16;
164  fr = (y*f) & 0xFFFF;
165  buf[y * stride + x ] += (color * (0x10000 - fr)) >> 16;
166  if(fr) buf[y * stride + x + 1] += (color * fr ) >> 16;
167  }
168  }
169 }
170 
171 /**
172  * Draw an arrow from (ex, ey) -> (sx, sy).
173  * @param w width of the image
174  * @param h height of the image
175  * @param stride stride/linesize of the image
176  * @param color color of the arrow
177  */
178 static void draw_arrow(uint8_t *buf, int sx, int sy, int ex,
179  int ey, int w, int h, int stride, int color, int tail, int direction)
180 {
181  int dx,dy;
182 
183  if (direction) {
184  FFSWAP(int, sx, ex);
185  FFSWAP(int, sy, ey);
186  }
187 
188  sx = av_clip(sx, -100, w + 100);
189  sy = av_clip(sy, -100, h + 100);
190  ex = av_clip(ex, -100, w + 100);
191  ey = av_clip(ey, -100, h + 100);
192 
193  dx = ex - sx;
194  dy = ey - sy;
195 
196  if (dx * dx + dy * dy > 3 * 3) {
197  int rx = dx + dy;
198  int ry = -dx + dy;
199  int length = sqrt((rx * rx + ry * ry) << 8);
200 
201  // FIXME subpixel accuracy
202  rx = ROUNDED_DIV(rx * (3 << 4), length);
203  ry = ROUNDED_DIV(ry * (3 << 4), length);
204 
205  if (tail) {
206  rx = -rx;
207  ry = -ry;
208  }
209 
210  draw_line(buf, sx, sy, sx + rx, sy + ry, w, h, stride, color);
211  draw_line(buf, sx, sy, sx - ry, sy + rx, w, h, stride, color);
212  }
213  draw_line(buf, sx, sy, ex, ey, w, h, stride, color);
214 }
215 
217 {
218  AVFilterContext *ctx = inlink->dst;
219  CodecViewContext *s = ctx->priv;
220  AVFilterLink *outlink = ctx->outputs[0];
221 
222  if (s->qp) {
223  int qstride, qp_type, ret;
224  int8_t *qp_table;
225 
226  ret = ff_qp_table_extract(frame, &qp_table, &qstride, NULL, &qp_type);
227  if (ret < 0) {
229  return ret;
230  }
231 
232  if (qp_table) {
233  int x, y;
234  const int w = AV_CEIL_RSHIFT(frame->width, s->hsub);
235  const int h = AV_CEIL_RSHIFT(frame->height, s->vsub);
236  uint8_t *pu = frame->data[1];
237  uint8_t *pv = frame->data[2];
238  const int lzu = frame->linesize[1];
239  const int lzv = frame->linesize[2];
240 
241  for (y = 0; y < h; y++) {
242  for (x = 0; x < w; x++) {
243  const int qp = ff_norm_qscale(qp_table[(y >> 3) * qstride + (x >> 3)], qp_type) * 128/31;
244  pu[x] = pv[x] = qp;
245  }
246  pu += lzu;
247  pv += lzv;
248  }
249  }
250  av_freep(&qp_table);
251  }
252 
253  if (s->mv || s->mv_type) {
255  if (sd) {
256  int i;
257  const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
258  const int is_iframe = (s->frame_type & FRAME_TYPE_I) && frame->pict_type == AV_PICTURE_TYPE_I;
259  const int is_pframe = (s->frame_type & FRAME_TYPE_P) && frame->pict_type == AV_PICTURE_TYPE_P;
260  const int is_bframe = (s->frame_type & FRAME_TYPE_B) && frame->pict_type == AV_PICTURE_TYPE_B;
261 
262  for (i = 0; i < sd->size / sizeof(*mvs); i++) {
263  const AVMotionVector *mv = &mvs[i];
264  const int direction = mv->source > 0;
265 
266  if (s->mv_type) {
267  const int is_fp = direction == 0 && (s->mv_type & MV_TYPE_FOR);
268  const int is_bp = direction == 1 && (s->mv_type & MV_TYPE_BACK);
269 
270  if ((!s->frame_type && (is_fp || is_bp)) ||
271  is_iframe && is_fp || is_iframe && is_bp ||
272  is_pframe && is_fp ||
273  is_bframe && is_fp || is_bframe && is_bp)
274  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
275  frame->width, frame->height, frame->linesize[0],
276  100, 0, direction);
277  } else if (s->mv)
278  if ((direction == 0 && (s->mv & MV_P_FOR) && frame->pict_type == AV_PICTURE_TYPE_P) ||
279  (direction == 0 && (s->mv & MV_B_FOR) && frame->pict_type == AV_PICTURE_TYPE_B) ||
280  (direction == 1 && (s->mv & MV_B_BACK) && frame->pict_type == AV_PICTURE_TYPE_B))
281  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
282  frame->width, frame->height, frame->linesize[0],
283  100, 0, direction);
284  }
285  }
286  }
287 
288  return ff_filter_frame(outlink, frame);
289 }
290 
292 {
293  AVFilterContext *ctx = inlink->dst;
294  CodecViewContext *s = ctx->priv;
296 
297  s->hsub = desc->log2_chroma_w;
298  s->vsub = desc->log2_chroma_h;
299  return 0;
300 }
301 
302 static const AVFilterPad codecview_inputs[] = {
303  {
304  .name = "default",
305  .type = AVMEDIA_TYPE_VIDEO,
306  .filter_frame = filter_frame,
307  .config_props = config_input,
308  .needs_writable = 1,
309  },
310  { NULL }
311 };
312 
313 static const AVFilterPad codecview_outputs[] = {
314  {
315  .name = "default",
316  .type = AVMEDIA_TYPE_VIDEO,
317  },
318  { NULL }
319 };
320 
322  .name = "codecview",
323  .description = NULL_IF_CONFIG_SMALL("Visualize information about some codecs."),
324  .priv_size = sizeof(CodecViewContext),
328  .priv_class = &codecview_class,
330 };
stride
int stride
Definition: mace.c:144
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
MV_B_FOR
#define MV_B_FOR
Definition: vf_codecview.c:40
av_clip
#define av_clip
Definition: common.h:122
qp_table.h
AVERROR
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later. That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another. Frame references ownership and permissions
opt.h
draw_line
static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color)
Draw a line from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:120
ff_make_format_list
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:286
FFSWAP
#define FFSWAP(type, a, b)
Definition: common.h:108
color
Definition: vf_paletteuse.c:583
av_frame_get_side_data
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:616
draw_arrow
static void draw_arrow(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color, int tail, int direction)
Draw an arrow from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:178
AVMotionVector
Definition: motion_vector.h:24
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:978
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2541
mv
static const int8_t mv[256][2]
Definition: 4xm.c:79
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:111
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:303
w
uint8_t w
Definition: llviddspenc.c:39
AVOption
AVOption.
Definition: opt.h:248
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:149
MV_P_FOR
#define MV_P_FOR
Definition: vf_codecview.c:39
query_formats
static int query_formats(AVFilterContext *ctx)
Definition: vf_codecview.c:81
CONST
#define CONST(name, help, val, unit)
Definition: vf_codecview.c:59
OFFSET
#define OFFSET(x)
Definition: vf_codecview.c:57
AVFilterFormats
A list of supported formats for one end of a filter link.
Definition: formats.h:65
CodecViewContext::hsub
int hsub
Definition: vf_codecview.c:53
clip_line
static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
Definition: vf_codecview.c:92
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:54
AVFrameSideData::size
size_t size
Definition: frame.h:212
inputs
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several inputs
Definition: filter_design.txt:243
ff_set_common_formats
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:580
CodecViewContext::mv_type
unsigned mv_type
Definition: vf_codecview.c:52
motion_vector.h
s
#define s(width, name)
Definition: cbs_vp9.c:257
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:58
outputs
static const AVFilterPad outputs[]
Definition: af_acontrast.c:203
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:290
ctx
AVFormatContext * ctx
Definition: movenc.c:48
ff_vf_codecview
const AVFilter ff_vf_codecview
Definition: vf_codecview.c:321
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
f
#define f(width, name)
Definition: cbs_vp9.c:255
FFABS
#define FFABS(a)
Absolute value, Note, INT_MIN / INT64_MIN result in undefined behavior as they are not representable ...
Definition: common.h:72
codecview_inputs
static const AVFilterPad codecview_inputs[]
Definition: vf_codecview.c:302
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:67
NULL
#define NULL
Definition: coverity.c:32
ROUNDED_DIV
#define ROUNDED_DIV(a, b)
Definition: common.h:56
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
codecview_options
static const AVOption codecview_options[]
Definition: vf_codecview.c:61
MV_B_BACK
#define MV_B_BACK
Definition: vf_codecview.c:41
color
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
AVFrameSideData::data
uint8_t * data
Definition: frame.h:211
CodecViewContext
Definition: vf_codecview.c:48
ff_norm_qscale
static int ff_norm_qscale(int qscale, int type)
Normalize the qscale factor FIXME the H264 qscale is a log based scale, mpeg1/2 is not,...
Definition: internal.h:351
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(codecview)
FRAME_TYPE_I
#define FRAME_TYPE_I
Definition: vf_codecview.c:44
codecview_outputs
static const AVFilterPad codecview_outputs[]
Definition: vf_codecview.c:313
internal.h
AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
#define AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
Some filters support a generic "enable" expression option that can be used to enable or disable a fil...
Definition: avfilter.h:126
CodecViewContext::vsub
int vsub
Definition: vf_codecview.c:53
i
int i
Definition: input.c:407
frame_type
frame_type
Definition: jpeg2000_parser.c:31
FRAME_TYPE_B
#define FRAME_TYPE_B
Definition: vf_codecview.c:46
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:60
pv
#define pv
Definition: regdef.h:60
AVFilter
Filter definition.
Definition: avfilter.h:145
ret
ret
Definition: filter_design.txt:187
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
filter_frame
static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_codecview.c:216
AV_PICTURE_TYPE_B
@ AV_PICTURE_TYPE_B
Bi-dir predicted.
Definition: avutil.h:276
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
avfilter.h
FLAGS
#define FLAGS
Definition: vf_codecview.c:58
AVFilterContext
An instance of a filter.
Definition: avfilter.h:333
desc
const char * desc
Definition: libsvtav1.c:79
AV_PICTURE_TYPE_P
@ AV_PICTURE_TYPE_P
Predicted.
Definition: avutil.h:275
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
MV_TYPE_FOR
#define MV_TYPE_FOR
Definition: vf_codecview.c:42
ff_qp_table_extract
int ff_qp_table_extract(AVFrame *frame, int8_t **table, int *table_w, int *table_h, int *qscale_type)
Extract a libpostproc-compatible QP table - an 8-bit QP value per 16x16 macroblock,...
Definition: qp_table.c:30
MV_TYPE_BACK
#define MV_TYPE_BACK
Definition: vf_codecview.c:43
AVFrameSideData
Structure to hold side data for an AVFrame.
Definition: frame.h:209
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:70
CodecViewContext::qp
int qp
Definition: vf_codecview.c:54
AV_OPT_TYPE_BOOL
@ AV_OPT_TYPE_BOOL
Definition: opt.h:242
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
CodecViewContext::mv
unsigned mv
Definition: vf_codecview.c:50
AV_OPT_TYPE_FLAGS
@ AV_OPT_TYPE_FLAGS
Definition: opt.h:224
imgutils.h
flags
#define flags(name, subs,...)
Definition: cbs_av1.c:561
AV_FRAME_DATA_MOTION_VECTORS
@ AV_FRAME_DATA_MOTION_VECTORS
Motion vectors exported by some codecs (on demand through the export_mvs flag set in the libavcodec A...
Definition: frame.h:96
config_input
static int config_input(AVFilterLink *inlink)
Definition: vf_codecview.c:291
h
h
Definition: vp9dsp_template.c:2038
CodecViewContext::frame_type
unsigned frame_type
Definition: vf_codecview.c:51
FRAME_TYPE_P
#define FRAME_TYPE_P
Definition: vf_codecview.c:45