FFmpeg
vf_codecview.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2002-2004 Michael Niedermayer <michaelni@gmx.at>
3  * Copyright (c) 2014 Clément Bœsch <u pkh me>
4  *
5  * This file is part of FFmpeg.
6  *
7  * FFmpeg is free software; you can redistribute it and/or
8  * modify it under the terms of the GNU Lesser General Public
9  * License as published by the Free Software Foundation; either
10  * version 2.1 of the License, or (at your option) any later version.
11  *
12  * FFmpeg is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
15  * Lesser General Public License for more details.
16  *
17  * You should have received a copy of the GNU Lesser General Public
18  * License along with FFmpeg; if not, write to the Free Software
19  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
20  */
21 
22 /**
23  * @file
24  * Codec debug viewer filter.
25  *
26  * All the MV drawing code from Michael Niedermayer is extracted from
27  * libavcodec/mpegvideo.c.
28  *
29  * TODO: segmentation
30  */
31 
32 #include "libavutil/imgutils.h"
34 #include "libavutil/opt.h"
35 #include "avfilter.h"
36 #include "qp_table.h"
37 #include "internal.h"
38 
39 #define MV_P_FOR (1<<0)
40 #define MV_B_FOR (1<<1)
41 #define MV_B_BACK (1<<2)
42 #define MV_TYPE_FOR (1<<0)
43 #define MV_TYPE_BACK (1<<1)
44 #define FRAME_TYPE_I (1<<0)
45 #define FRAME_TYPE_P (1<<1)
46 #define FRAME_TYPE_B (1<<2)
47 
48 typedef struct CodecViewContext {
49  const AVClass *class;
50  unsigned mv;
51  unsigned frame_type;
52  unsigned mv_type;
53  int hsub, vsub;
54  int qp;
56 
57 #define OFFSET(x) offsetof(CodecViewContext, x)
58 #define FLAGS AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_VIDEO_PARAM
59 #define CONST(name, help, val, unit) { name, help, 0, AV_OPT_TYPE_CONST, {.i64=val}, 0, 0, FLAGS, unit }
60 
61 static const AVOption codecview_options[] = {
62  { "mv", "set motion vectors to visualize", OFFSET(mv), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv" },
63  CONST("pf", "forward predicted MVs of P-frames", MV_P_FOR, "mv"),
64  CONST("bf", "forward predicted MVs of B-frames", MV_B_FOR, "mv"),
65  CONST("bb", "backward predicted MVs of B-frames", MV_B_BACK, "mv"),
66  { "qp", NULL, OFFSET(qp), AV_OPT_TYPE_BOOL, {.i64=0}, 0, 1, .flags = FLAGS },
67  { "mv_type", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
68  { "mvt", "set motion vectors type", OFFSET(mv_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "mv_type" },
69  CONST("fp", "forward predicted MVs", MV_TYPE_FOR, "mv_type"),
70  CONST("bp", "backward predicted MVs", MV_TYPE_BACK, "mv_type"),
71  { "frame_type", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
72  { "ft", "set frame types to visualize motion vectors of", OFFSET(frame_type), AV_OPT_TYPE_FLAGS, {.i64=0}, 0, INT_MAX, FLAGS, "frame_type" },
73  CONST("if", "I-frames", FRAME_TYPE_I, "frame_type"),
74  CONST("pf", "P-frames", FRAME_TYPE_P, "frame_type"),
75  CONST("bf", "B-frames", FRAME_TYPE_B, "frame_type"),
76  { NULL }
77 };
78 
79 AVFILTER_DEFINE_CLASS(codecview);
80 
82 {
83  // TODO: we can probably add way more pixel formats without any other
84  // changes; anything with 8-bit luma in first plane should be working
87 }
88 
89 static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
90 {
91  if(*sx > *ex)
92  return clip_line(ex, ey, sx, sy, maxx);
93 
94  if (*sx < 0) {
95  if (*ex < 0)
96  return 1;
97  *sy = *ey + (*sy - *ey) * (int64_t)*ex / (*ex - *sx);
98  *sx = 0;
99  }
100 
101  if (*ex > maxx) {
102  if (*sx > maxx)
103  return 1;
104  *ey = *sy + (*ey - *sy) * (int64_t)(maxx - *sx) / (*ex - *sx);
105  *ex = maxx;
106  }
107  return 0;
108 }
109 
110 /**
111  * Draw a line from (ex, ey) -> (sx, sy).
112  * @param w width of the image
113  * @param h height of the image
114  * @param stride stride/linesize of the image
115  * @param color color of the arrow
116  */
117 static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey,
118  int w, int h, int stride, int color)
119 {
120  int x, y, fr, f;
121 
122  if (clip_line(&sx, &sy, &ex, &ey, w - 1))
123  return;
124  if (clip_line(&sy, &sx, &ey, &ex, h - 1))
125  return;
126 
127  sx = av_clip(sx, 0, w - 1);
128  sy = av_clip(sy, 0, h - 1);
129  ex = av_clip(ex, 0, w - 1);
130  ey = av_clip(ey, 0, h - 1);
131 
132  buf[sy * stride + sx] += color;
133 
134  if (FFABS(ex - sx) > FFABS(ey - sy)) {
135  if (sx > ex) {
136  FFSWAP(int, sx, ex);
137  FFSWAP(int, sy, ey);
138  }
139  buf += sx + sy * stride;
140  ex -= sx;
141  f = ((ey - sy) * (1 << 16)) / ex;
142  for (x = 0; x <= ex; x++) {
143  y = (x * f) >> 16;
144  fr = (x * f) & 0xFFFF;
145  buf[ y * stride + x] += (color * (0x10000 - fr)) >> 16;
146  if(fr) buf[(y + 1) * stride + x] += (color * fr ) >> 16;
147  }
148  } else {
149  if (sy > ey) {
150  FFSWAP(int, sx, ex);
151  FFSWAP(int, sy, ey);
152  }
153  buf += sx + sy * stride;
154  ey -= sy;
155  if (ey)
156  f = ((ex - sx) * (1 << 16)) / ey;
157  else
158  f = 0;
159  for(y= 0; y <= ey; y++){
160  x = (y*f) >> 16;
161  fr = (y*f) & 0xFFFF;
162  buf[y * stride + x ] += (color * (0x10000 - fr)) >> 16;
163  if(fr) buf[y * stride + x + 1] += (color * fr ) >> 16;
164  }
165  }
166 }
167 
168 /**
169  * Draw an arrow from (ex, ey) -> (sx, sy).
170  * @param w width of the image
171  * @param h height of the image
172  * @param stride stride/linesize of the image
173  * @param color color of the arrow
174  */
175 static void draw_arrow(uint8_t *buf, int sx, int sy, int ex,
176  int ey, int w, int h, int stride, int color, int tail, int direction)
177 {
178  int dx,dy;
179 
180  if (direction) {
181  FFSWAP(int, sx, ex);
182  FFSWAP(int, sy, ey);
183  }
184 
185  sx = av_clip(sx, -100, w + 100);
186  sy = av_clip(sy, -100, h + 100);
187  ex = av_clip(ex, -100, w + 100);
188  ey = av_clip(ey, -100, h + 100);
189 
190  dx = ex - sx;
191  dy = ey - sy;
192 
193  if (dx * dx + dy * dy > 3 * 3) {
194  int rx = dx + dy;
195  int ry = -dx + dy;
196  int length = sqrt((rx * rx + ry * ry) << 8);
197 
198  // FIXME subpixel accuracy
199  rx = ROUNDED_DIV(rx * (3 << 4), length);
200  ry = ROUNDED_DIV(ry * (3 << 4), length);
201 
202  if (tail) {
203  rx = -rx;
204  ry = -ry;
205  }
206 
207  draw_line(buf, sx, sy, sx + rx, sy + ry, w, h, stride, color);
208  draw_line(buf, sx, sy, sx - ry, sy + rx, w, h, stride, color);
209  }
210  draw_line(buf, sx, sy, ex, ey, w, h, stride, color);
211 }
212 
214 {
215  AVFilterContext *ctx = inlink->dst;
216  CodecViewContext *s = ctx->priv;
217  AVFilterLink *outlink = ctx->outputs[0];
218 
219  if (s->qp) {
220  int qstride, qp_type, ret;
221  int8_t *qp_table;
222 
223  ret = ff_qp_table_extract(frame, &qp_table, &qstride, NULL, &qp_type);
224  if (ret < 0) {
226  return ret;
227  }
228 
229  if (qp_table) {
230  int x, y;
231  const int w = AV_CEIL_RSHIFT(frame->width, s->hsub);
232  const int h = AV_CEIL_RSHIFT(frame->height, s->vsub);
233  uint8_t *pu = frame->data[1];
234  uint8_t *pv = frame->data[2];
235  const int lzu = frame->linesize[1];
236  const int lzv = frame->linesize[2];
237 
238  for (y = 0; y < h; y++) {
239  for (x = 0; x < w; x++) {
240  const int qp = ff_norm_qscale(qp_table[(y >> 3) * qstride + (x >> 3)], qp_type) * 128/31;
241  pu[x] = pv[x] = qp;
242  }
243  pu += lzu;
244  pv += lzv;
245  }
246  }
247  av_freep(&qp_table);
248  }
249 
250  if (s->mv || s->mv_type) {
252  if (sd) {
253  int i;
254  const AVMotionVector *mvs = (const AVMotionVector *)sd->data;
255  const int is_iframe = (s->frame_type & FRAME_TYPE_I) && frame->pict_type == AV_PICTURE_TYPE_I;
256  const int is_pframe = (s->frame_type & FRAME_TYPE_P) && frame->pict_type == AV_PICTURE_TYPE_P;
257  const int is_bframe = (s->frame_type & FRAME_TYPE_B) && frame->pict_type == AV_PICTURE_TYPE_B;
258 
259  for (i = 0; i < sd->size / sizeof(*mvs); i++) {
260  const AVMotionVector *mv = &mvs[i];
261  const int direction = mv->source > 0;
262 
263  if (s->mv_type) {
264  const int is_fp = direction == 0 && (s->mv_type & MV_TYPE_FOR);
265  const int is_bp = direction == 1 && (s->mv_type & MV_TYPE_BACK);
266 
267  if ((!s->frame_type && (is_fp || is_bp)) ||
268  is_iframe && is_fp || is_iframe && is_bp ||
269  is_pframe && is_fp ||
270  is_bframe && is_fp || is_bframe && is_bp)
271  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
272  frame->width, frame->height, frame->linesize[0],
273  100, 0, direction);
274  } else if (s->mv)
275  if ((direction == 0 && (s->mv & MV_P_FOR) && frame->pict_type == AV_PICTURE_TYPE_P) ||
276  (direction == 0 && (s->mv & MV_B_FOR) && frame->pict_type == AV_PICTURE_TYPE_B) ||
277  (direction == 1 && (s->mv & MV_B_BACK) && frame->pict_type == AV_PICTURE_TYPE_B))
278  draw_arrow(frame->data[0], mv->dst_x, mv->dst_y, mv->src_x, mv->src_y,
279  frame->width, frame->height, frame->linesize[0],
280  100, 0, direction);
281  }
282  }
283  }
284 
285  return ff_filter_frame(outlink, frame);
286 }
287 
289 {
290  AVFilterContext *ctx = inlink->dst;
291  CodecViewContext *s = ctx->priv;
293 
294  s->hsub = desc->log2_chroma_w;
295  s->vsub = desc->log2_chroma_h;
296  return 0;
297 }
298 
299 static const AVFilterPad codecview_inputs[] = {
300  {
301  .name = "default",
302  .type = AVMEDIA_TYPE_VIDEO,
304  .filter_frame = filter_frame,
305  .config_props = config_input,
306  },
307 };
308 
309 static const AVFilterPad codecview_outputs[] = {
310  {
311  .name = "default",
312  .type = AVMEDIA_TYPE_VIDEO,
313  },
314 };
315 
317  .name = "codecview",
318  .description = NULL_IF_CONFIG_SMALL("Visualize information about some codecs."),
319  .priv_size = sizeof(CodecViewContext),
323  .priv_class = &codecview_class,
325 };
stride
int stride
Definition: mace.c:144
AVPixelFormat
AVPixelFormat
Pixel format.
Definition: pixfmt.h:64
MV_B_FOR
#define MV_B_FOR
Definition: vf_codecview.c:40
av_clip
#define av_clip
Definition: common.h:96
qp_table.h
opt.h
draw_line
static void draw_line(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color)
Draw a line from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:117
color
Definition: vf_paletteuse.c:587
av_frame_get_side_data
AVFrameSideData * av_frame_get_side_data(const AVFrame *frame, enum AVFrameSideDataType type)
Definition: frame.c:617
draw_arrow
static void draw_arrow(uint8_t *buf, int sx, int sy, int ex, int ey, int w, int h, int stride, int color, int tail, int direction)
Draw an arrow from (ex, ey) -> (sx, sy).
Definition: vf_codecview.c:175
AVMotionVector
Definition: motion_vector.h:24
ff_filter_frame
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1017
av_pix_fmt_desc_get
const AVPixFmtDescriptor * av_pix_fmt_desc_get(enum AVPixelFormat pix_fmt)
Definition: pixdesc.c:2540
mv
static const int8_t mv[256][2]
Definition: 4xm.c:79
inlink
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
Definition: filter_design.txt:212
av_frame_free
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:112
AVFrame
This structure describes decoded (raw) audio or video data.
Definition: frame.h:303
w
uint8_t w
Definition: llviddspenc.c:38
AVOption
AVOption.
Definition: opt.h:247
AVFilter::name
const char * name
Filter name.
Definition: avfilter.h:153
MV_P_FOR
#define MV_P_FOR
Definition: vf_codecview.c:39
query_formats
static int query_formats(AVFilterContext *ctx)
Definition: vf_codecview.c:81
CONST
#define CONST(name, help, val, unit)
Definition: vf_codecview.c:59
OFFSET
#define OFFSET(x)
Definition: vf_codecview.c:57
CodecViewContext::hsub
int hsub
Definition: vf_codecview.c:53
clip_line
static int clip_line(int *sx, int *sy, int *ex, int *ey, int maxx)
Definition: vf_codecview.c:89
ff_norm_qscale
static int ff_norm_qscale(int qscale, int type)
Normalize the qscale factor FIXME the H264 qscale is a log based scale, mpeg1/2 is not,...
Definition: qp_table.h:39
AVFilterPad
A filter pad used for either input or output.
Definition: internal.h:50
AVFrameSideData::size
size_t size
Definition: frame.h:212
CodecViewContext::mv_type
unsigned mv_type
Definition: vf_codecview.c:52
motion_vector.h
s
#define s(width, name)
Definition: cbs_vp9.c:257
AV_CEIL_RSHIFT
#define AV_CEIL_RSHIFT(a, b)
Definition: common.h:51
ff_set_common_formats_from_list
int ff_set_common_formats_from_list(AVFilterContext *ctx, const int *fmts)
Equivalent to ff_set_common_formats(ctx, ff_make_format_list(fmts))
Definition: formats.c:703
pix_fmts
static enum AVPixelFormat pix_fmts[]
Definition: libkvazaar.c:290
ctx
AVFormatContext * ctx
Definition: movenc.c:48
ff_vf_codecview
const AVFilter ff_vf_codecview
Definition: vf_codecview.c:316
AV_PIX_FMT_YUV420P
@ AV_PIX_FMT_YUV420P
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
f
#define f(width, name)
Definition: cbs_vp9.c:255
FILTER_INPUTS
#define FILTER_INPUTS(array)
Definition: internal.h:152
FFABS
#define FFABS(a)
Absolute value, Note, INT_MIN / INT64_MIN result in undefined behavior as they are not representable ...
Definition: common.h:65
codecview_inputs
static const AVFilterPad codecview_inputs[]
Definition: vf_codecview.c:299
AVClass
Describe the class of an AVClass context structure.
Definition: log.h:66
NULL
#define NULL
Definition: coverity.c:32
ROUNDED_DIV
#define ROUNDED_DIV(a, b)
Definition: common.h:49
AV_PICTURE_TYPE_I
@ AV_PICTURE_TYPE_I
Intra.
Definition: avutil.h:274
NULL_IF_CONFIG_SMALL
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification.
Definition: internal.h:117
codecview_options
static const AVOption codecview_options[]
Definition: vf_codecview.c:61
MV_B_BACK
#define MV_B_BACK
Definition: vf_codecview.c:41
color
static const uint32_t color[16+AV_CLASS_CATEGORY_NB]
Definition: log.c:92
AVFrameSideData::data
uint8_t * data
Definition: frame.h:211
CodecViewContext
Definition: vf_codecview.c:48
AVFILTER_DEFINE_CLASS
AVFILTER_DEFINE_CLASS(codecview)
FRAME_TYPE_I
#define FRAME_TYPE_I
Definition: vf_codecview.c:44
codecview_outputs
static const AVFilterPad codecview_outputs[]
Definition: vf_codecview.c:309
internal.h
AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
#define AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC
Some filters support a generic "enable" expression option that can be used to enable or disable a fil...
Definition: avfilter.h:130
CodecViewContext::vsub
int vsub
Definition: vf_codecview.c:53
i
int i
Definition: input.c:406
frame_type
frame_type
Definition: jpeg2000_parser.c:31
FRAME_TYPE_B
#define FRAME_TYPE_B
Definition: vf_codecview.c:46
AVFilterPad::name
const char * name
Pad name.
Definition: internal.h:56
pv
#define pv
Definition: regdef.h:60
AVFilter
Filter definition.
Definition: avfilter.h:149
ret
ret
Definition: filter_design.txt:187
FFSWAP
#define FFSWAP(type, a, b)
Definition: macros.h:52
frame
these buffered frames must be flushed immediately if a new input produces new the filter must not call request_frame to get more It must just process the frame or queue it The task of requesting more frames is left to the filter s request_frame method or the application If a filter has several the filter must be ready for frames arriving randomly on any input any filter with several inputs will most likely require some kind of queuing mechanism It is perfectly acceptable to have a limited queue and to drop frames when the inputs are too unbalanced request_frame For filters that do not use the this method is called when a frame is wanted on an output For a it should directly call filter_frame on the corresponding output For a if there are queued frames already one of these frames should be pushed If the filter should request a frame on one of its repeatedly until at least one frame has been pushed Return or at least make progress towards producing a frame
Definition: filter_design.txt:264
filter_frame
static int filter_frame(AVFilterLink *inlink, AVFrame *frame)
Definition: vf_codecview.c:213
AV_PICTURE_TYPE_B
@ AV_PICTURE_TYPE_B
Bi-dir predicted.
Definition: avutil.h:276
AV_PIX_FMT_NONE
@ AV_PIX_FMT_NONE
Definition: pixfmt.h:65
avfilter.h
FLAGS
#define FLAGS
Definition: vf_codecview.c:58
AVFilterContext
An instance of a filter.
Definition: avfilter.h:346
desc
const char * desc
Definition: libsvtav1.c:79
AV_PICTURE_TYPE_P
@ AV_PICTURE_TYPE_P
Predicted.
Definition: avutil.h:275
AVMEDIA_TYPE_VIDEO
@ AVMEDIA_TYPE_VIDEO
Definition: avutil.h:201
MV_TYPE_FOR
#define MV_TYPE_FOR
Definition: vf_codecview.c:42
ff_qp_table_extract
int ff_qp_table_extract(AVFrame *frame, int8_t **table, int *table_w, int *table_h, int *qscale_type)
Extract a libpostproc-compatible QP table - an 8-bit QP value per 16x16 macroblock,...
Definition: qp_table.c:30
MV_TYPE_BACK
#define MV_TYPE_BACK
Definition: vf_codecview.c:43
AVFrameSideData
Structure to hold side data for an AVFrame.
Definition: frame.h:209
AVPixFmtDescriptor
Descriptor that unambiguously describes how the bits of a pixel are stored in the up to 4 data planes...
Definition: pixdesc.h:69
CodecViewContext::qp
int qp
Definition: vf_codecview.c:54
AV_OPT_TYPE_BOOL
@ AV_OPT_TYPE_BOOL
Definition: opt.h:241
FILTER_OUTPUTS
#define FILTER_OUTPUTS(array)
Definition: internal.h:153
av_freep
#define av_freep(p)
Definition: tableprint_vlc.h:35
CodecViewContext::mv
unsigned mv
Definition: vf_codecview.c:50
AV_OPT_TYPE_FLAGS
@ AV_OPT_TYPE_FLAGS
Definition: opt.h:223
imgutils.h
AV_FRAME_DATA_MOTION_VECTORS
@ AV_FRAME_DATA_MOTION_VECTORS
Motion vectors exported by some codecs (on demand through the export_mvs flag set in the libavcodec A...
Definition: frame.h:96
config_input
static int config_input(AVFilterLink *inlink)
Definition: vf_codecview.c:288
h
h
Definition: vp9dsp_template.c:2038
CodecViewContext::frame_type
unsigned frame_type
Definition: vf_codecview.c:51
FRAME_TYPE_P
#define FRAME_TYPE_P
Definition: vf_codecview.c:45
AVFILTERPAD_FLAG_NEEDS_WRITABLE
#define AVFILTERPAD_FLAG_NEEDS_WRITABLE
The filter expects writable frames from its input link, duplicating data buffers if needed.
Definition: internal.h:69