FFmpeg
af_anlmdn.c
Go to the documentation of this file.
1 /*
2  * Copyright (c) 2019 Paul B Mahol
3  *
4  * This file is part of FFmpeg.
5  *
6  * FFmpeg is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU Lesser General Public
8  * License as published by the Free Software Foundation; either
9  * version 2.1 of the License, or (at your option) any later version.
10  *
11  * FFmpeg is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14  * Lesser General Public License for more details.
15  *
16  * You should have received a copy of the GNU Lesser General Public
17  * License along with FFmpeg; if not, write to the Free Software
18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
19  */
20 
21 #include <float.h>
22 
23 #include "libavutil/avassert.h"
24 #include "libavutil/audio_fifo.h"
25 #include "libavutil/avstring.h"
26 #include "libavutil/opt.h"
27 #include "avfilter.h"
28 #include "audio.h"
29 #include "formats.h"
30 
31 #include "af_anlmdndsp.h"
32 
33 #define WEIGHT_LUT_NBITS 20
34 #define WEIGHT_LUT_SIZE (1<<WEIGHT_LUT_NBITS)
35 
36 #define SQR(x) ((x) * (x))
37 
38 typedef struct AudioNLMeansContext {
39  const AVClass *class;
40 
41  float a;
42  int64_t pd;
43  int64_t rd;
44  float m;
45  int om;
46 
49 
50  int K;
51  int S;
52  int N;
53  int H;
54 
55  int offset;
58 
59  int64_t pts;
60 
62  int eof_left;
63 
66 
67 enum OutModes {
72 };
73 
74 #define OFFSET(x) offsetof(AudioNLMeansContext, x)
75 #define AFT AV_OPT_FLAG_AUDIO_PARAM|AV_OPT_FLAG_FILTERING_PARAM|AV_OPT_FLAG_RUNTIME_PARAM
76 
77 static const AVOption anlmdn_options[] = {
78  { "s", "set denoising strength", OFFSET(a), AV_OPT_TYPE_FLOAT, {.dbl=0.00001},0.00001, 10, AFT },
79  { "p", "set patch duration", OFFSET(pd), AV_OPT_TYPE_DURATION, {.i64=2000}, 1000, 100000, AFT },
80  { "r", "set research duration", OFFSET(rd), AV_OPT_TYPE_DURATION, {.i64=6000}, 2000, 300000, AFT },
81  { "o", "set output mode", OFFSET(om), AV_OPT_TYPE_INT, {.i64=OUT_MODE}, 0, NB_MODES-1, AFT, "mode" },
82  { "i", "input", 0, AV_OPT_TYPE_CONST, {.i64=IN_MODE}, 0, 0, AFT, "mode" },
83  { "o", "output", 0, AV_OPT_TYPE_CONST, {.i64=OUT_MODE}, 0, 0, AFT, "mode" },
84  { "n", "noise", 0, AV_OPT_TYPE_CONST, {.i64=NOISE_MODE},0, 0, AFT, "mode" },
85  { "m", "set smooth factor", OFFSET(m), AV_OPT_TYPE_FLOAT, {.dbl=11.}, 1, 15, AFT },
86  { NULL }
87 };
88 
89 AVFILTER_DEFINE_CLASS(anlmdn);
90 
92 {
95  static const enum AVSampleFormat sample_fmts[] = {
98  };
99  int ret;
100 
101  formats = ff_make_format_list(sample_fmts);
102  if (!formats)
103  return AVERROR(ENOMEM);
104  ret = ff_set_common_formats(ctx, formats);
105  if (ret < 0)
106  return ret;
107 
108  layouts = ff_all_channel_counts();
109  if (!layouts)
110  return AVERROR(ENOMEM);
111 
112  ret = ff_set_common_channel_layouts(ctx, layouts);
113  if (ret < 0)
114  return ret;
115 
116  formats = ff_all_samplerates();
117  return ff_set_common_samplerates(ctx, formats);
118 }
119 
120 static float compute_distance_ssd_c(const float *f1, const float *f2, ptrdiff_t K)
121 {
122  float distance = 0.;
123 
124  for (int k = -K; k <= K; k++)
125  distance += SQR(f1[k] - f2[k]);
126 
127  return distance;
128 }
129 
130 static void compute_cache_c(float *cache, const float *f,
131  ptrdiff_t S, ptrdiff_t K,
132  ptrdiff_t i, ptrdiff_t jj)
133 {
134  int v = 0;
135 
136  for (int j = jj; j < jj + S; j++, v++)
137  cache[v] += -SQR(f[i - K - 1] - f[j - K - 1]) + SQR(f[i + K] - f[j + K]);
138 }
139 
141 {
144 
145  if (ARCH_X86)
146  ff_anlmdn_init_x86(dsp);
147 }
148 
150 {
151  AudioNLMeansContext *s = ctx->priv;
152  AVFilterLink *outlink = ctx->outputs[0];
153  int newK, newS, newH, newN;
154  AVFrame *new_in, *new_cache;
155 
156  newK = av_rescale(s->pd, outlink->sample_rate, AV_TIME_BASE);
157  newS = av_rescale(s->rd, outlink->sample_rate, AV_TIME_BASE);
158 
159  newH = newK * 2 + 1;
160  newN = newH + (newK + newS) * 2;
161 
162  av_log(ctx, AV_LOG_DEBUG, "K:%d S:%d H:%d N:%d\n", newK, newS, newH, newN);
163 
164  if (!s->cache || s->cache->nb_samples < newS * 2) {
165  new_cache = ff_get_audio_buffer(outlink, newS * 2);
166  if (new_cache) {
167  av_frame_free(&s->cache);
168  s->cache = new_cache;
169  } else {
170  return AVERROR(ENOMEM);
171  }
172  }
173  if (!s->cache)
174  return AVERROR(ENOMEM);
175 
176  s->pdiff_lut_scale = 1.f / s->m * WEIGHT_LUT_SIZE;
177  for (int i = 0; i < WEIGHT_LUT_SIZE; i++) {
178  float w = -i / s->pdiff_lut_scale;
179 
180  s->weight_lut[i] = expf(w);
181  }
182 
183  if (!s->in || s->in->nb_samples < newN) {
184  new_in = ff_get_audio_buffer(outlink, newN);
185  if (new_in) {
186  av_frame_free(&s->in);
187  s->in = new_in;
188  } else {
189  return AVERROR(ENOMEM);
190  }
191  }
192  if (!s->in)
193  return AVERROR(ENOMEM);
194 
195  s->K = newK;
196  s->S = newS;
197  s->H = newH;
198  s->N = newN;
199 
200  return 0;
201 }
202 
203 static int config_output(AVFilterLink *outlink)
204 {
205  AVFilterContext *ctx = outlink->src;
206  AudioNLMeansContext *s = ctx->priv;
207  int ret;
208 
209  s->eof_left = -1;
210  s->pts = AV_NOPTS_VALUE;
211 
212  ret = config_filter(ctx);
213  if (ret < 0)
214  return ret;
215 
216  s->fifo = av_audio_fifo_alloc(outlink->format, outlink->channels, s->N);
217  if (!s->fifo)
218  return AVERROR(ENOMEM);
219 
220  ret = av_audio_fifo_write(s->fifo, (void **)s->in->extended_data, s->K + s->S);
221  if (ret < 0)
222  return ret;
223 
224  ff_anlmdn_init(&s->dsp);
225 
226  return 0;
227 }
228 
229 static int filter_channel(AVFilterContext *ctx, void *arg, int ch, int nb_jobs)
230 {
231  AudioNLMeansContext *s = ctx->priv;
232  AVFrame *out = arg;
233  const int S = s->S;
234  const int K = s->K;
235  const int om = s->om;
236  const float *f = (const float *)(s->in->extended_data[ch]) + K;
237  float *cache = (float *)s->cache->extended_data[ch];
238  const float sw = (65536.f / (4 * K + 2)) / sqrtf(s->a);
239  float *dst = (float *)out->extended_data[ch] + s->offset;
240  const float smooth = s->m;
241 
242  for (int i = S; i < s->H + S; i++) {
243  float P = 0.f, Q = 0.f;
244  int v = 0;
245 
246  if (i == S) {
247  for (int j = i - S; j <= i + S; j++) {
248  if (i == j)
249  continue;
250  cache[v++] = s->dsp.compute_distance_ssd(f + i, f + j, K);
251  }
252  } else {
253  s->dsp.compute_cache(cache, f, S, K, i, i - S);
254  s->dsp.compute_cache(cache + S, f, S, K, i, i + 1);
255  }
256 
257  for (int j = 0; j < 2 * S && !ctx->is_disabled; j++) {
258  const float distance = cache[j];
259  unsigned weight_lut_idx;
260  float w;
261 
262  if (distance < 0.f) {
263  cache[j] = 0.f;
264  continue;
265  }
266  w = distance * sw;
267  if (w >= smooth)
268  continue;
269  weight_lut_idx = w * s->pdiff_lut_scale;
270  av_assert2(weight_lut_idx < WEIGHT_LUT_SIZE);
271  w = s->weight_lut[weight_lut_idx];
272  P += w * f[i - S + j + (j >= S)];
273  Q += w;
274  }
275 
276  P += f[i];
277  Q += 1;
278 
279  switch (om) {
280  case IN_MODE: dst[i - S] = f[i]; break;
281  case OUT_MODE: dst[i - S] = P / Q; break;
282  case NOISE_MODE: dst[i - S] = f[i] - (P / Q); break;
283  }
284  }
285 
286  return 0;
287 }
288 
290 {
291  AVFilterContext *ctx = inlink->dst;
292  AVFilterLink *outlink = ctx->outputs[0];
293  AudioNLMeansContext *s = ctx->priv;
294  AVFrame *out = NULL;
295  int available, wanted, ret;
296 
297  if (s->pts == AV_NOPTS_VALUE)
298  s->pts = in->pts;
299 
300  ret = av_audio_fifo_write(s->fifo, (void **)in->extended_data,
301  in->nb_samples);
302  av_frame_free(&in);
303 
304  s->offset = 0;
305  available = av_audio_fifo_size(s->fifo);
306  wanted = (available / s->H) * s->H;
307 
308  if (wanted >= s->H && available >= s->N) {
309  out = ff_get_audio_buffer(outlink, wanted);
310  if (!out)
311  return AVERROR(ENOMEM);
312  }
313 
314  while (available >= s->N) {
315  ret = av_audio_fifo_peek(s->fifo, (void **)s->in->extended_data, s->N);
316  if (ret < 0)
317  break;
318 
319  ctx->internal->execute(ctx, filter_channel, out, NULL, inlink->channels);
320 
321  av_audio_fifo_drain(s->fifo, s->H);
322 
323  s->offset += s->H;
324  available -= s->H;
325  }
326 
327  if (out) {
328  out->pts = s->pts;
329  out->nb_samples = s->offset;
330  if (s->eof_left >= 0) {
331  out->nb_samples = FFMIN(s->eof_left, s->offset);
332  s->eof_left -= out->nb_samples;
333  }
334  s->pts += av_rescale_q(s->offset, (AVRational){1, outlink->sample_rate}, outlink->time_base);
335 
336  return ff_filter_frame(outlink, out);
337  }
338 
339  return ret;
340 }
341 
342 static int request_frame(AVFilterLink *outlink)
343 {
344  AVFilterContext *ctx = outlink->src;
345  AudioNLMeansContext *s = ctx->priv;
346  int ret;
347 
348  ret = ff_request_frame(ctx->inputs[0]);
349 
350  if (ret == AVERROR_EOF && s->eof_left != 0) {
351  AVFrame *in;
352 
353  if (s->eof_left < 0)
354  s->eof_left = av_audio_fifo_size(s->fifo) - (s->S + s->K);
355  if (s->eof_left <= 0)
356  return AVERROR_EOF;
357  in = ff_get_audio_buffer(outlink, s->H);
358  if (!in)
359  return AVERROR(ENOMEM);
360 
361  return filter_frame(ctx->inputs[0], in);
362  }
363 
364  return ret;
365 }
366 
367 static int process_command(AVFilterContext *ctx, const char *cmd, const char *args,
368  char *res, int res_len, int flags)
369 {
370  int ret;
371 
372  ret = ff_filter_process_command(ctx, cmd, args, res, res_len, flags);
373  if (ret < 0)
374  return ret;
375 
376  ret = config_filter(ctx);
377  if (ret < 0)
378  return ret;
379 
380  return 0;
381 }
382 
384 {
385  AudioNLMeansContext *s = ctx->priv;
386 
388  av_frame_free(&s->in);
389  av_frame_free(&s->cache);
390 }
391 
392 static const AVFilterPad inputs[] = {
393  {
394  .name = "default",
395  .type = AVMEDIA_TYPE_AUDIO,
396  .filter_frame = filter_frame,
397  },
398  { NULL }
399 };
400 
401 static const AVFilterPad outputs[] = {
402  {
403  .name = "default",
404  .type = AVMEDIA_TYPE_AUDIO,
405  .config_props = config_output,
406  .request_frame = request_frame,
407  },
408  { NULL }
409 };
410 
412  .name = "anlmdn",
413  .description = NULL_IF_CONFIG_SMALL("Reduce broadband noise from stream using Non-Local Means."),
414  .query_formats = query_formats,
415  .priv_size = sizeof(AudioNLMeansContext),
416  .priv_class = &anlmdn_class,
417  .uninit = uninit,
418  .inputs = inputs,
419  .outputs = outputs,
423 };
static const AVOption anlmdn_options[]
Definition: af_anlmdn.c:77
float, planar
Definition: samplefmt.h:69
#define NULL
Definition: coverity.c:32
#define P
AVAudioFifo * av_audio_fifo_alloc(enum AVSampleFormat sample_fmt, int channels, int nb_samples)
Allocate an AVAudioFifo.
Definition: audio_fifo.c:59
This structure describes decoded (raw) audio or video data.
Definition: frame.h:314
static int process_command(AVFilterContext *ctx, const char *cmd, const char *args, char *res, int res_len, int flags)
Definition: af_anlmdn.c:367
AVOption.
Definition: opt.h:248
static float compute_distance_ssd_c(const float *f1, const float *f2, ptrdiff_t K)
Definition: af_anlmdn.c:120
Main libavfilter public API header.
void av_audio_fifo_free(AVAudioFifo *af)
Free an AVAudioFifo.
Definition: audio_fifo.c:45
#define AFT
Definition: af_anlmdn.c:75
int ff_set_common_channel_layouts(AVFilterContext *ctx, AVFilterChannelLayouts *channel_layouts)
A helper for query_formats() which sets all links to the same list of channel layouts/sample rates...
Definition: formats.c:569
#define SQR(x)
Definition: af_anlmdn.c:36
void ff_anlmdn_init(AudioNLMDNDSPContext *dsp)
Definition: af_anlmdn.c:140
int is_disabled
the enabled state from the last expression evaluation
Definition: avfilter.h:388
AVFilterFormats * ff_make_format_list(const int *fmts)
Create a list of supported formats.
Definition: formats.c:287
const char * name
Pad name.
Definition: internal.h:60
AVFilterLink ** inputs
array of pointers to input links
Definition: avfilter.h:349
int ff_filter_frame(AVFilterLink *link, AVFrame *frame)
Send a frame of data to the next filter.
Definition: avfilter.c:1093
#define av_cold
Definition: attributes.h:88
#define av_assert2(cond)
assert() equivalent, that does lie in speed critical code.
Definition: avassert.h:64
AVOptions.
#define f(width, name)
Definition: cbs_vp9.c:255
AVFilter ff_af_anlmdn
Definition: af_anlmdn.c:411
int64_t pts
Presentation timestamp in time_base units (time when frame should be shown to user).
Definition: frame.h:407
#define AVERROR_EOF
End of file.
Definition: error.h:55
static int config_output(AVFilterLink *outlink)
Definition: af_anlmdn.c:203
float(* compute_distance_ssd)(const float *f1, const float *f2, ptrdiff_t K)
Definition: af_anlmdndsp.h:32
#define av_log(a,...)
A filter pad used for either input or output.
Definition: internal.h:54
int64_t av_rescale_q(int64_t a, AVRational bq, AVRational cq)
Rescale a 64-bit integer by 2 rational numbers.
Definition: mathematics.c:142
#define expf(x)
Definition: libm.h:283
int ff_set_common_formats(AVFilterContext *ctx, AVFilterFormats *formats)
A helper for query_formats() which sets all links to the same list of formats.
Definition: formats.c:588
if no frame is available
AVFrame * ff_get_audio_buffer(AVFilterLink *link, int nb_samples)
Request an audio samples buffer with a specific set of permissions.
Definition: audio.c:86
void av_frame_free(AVFrame **frame)
Free the frame and any dynamically allocated objects in it, e.g.
Definition: frame.c:203
void ff_anlmdn_init_x86(AudioNLMDNDSPContext *s)
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:115
int ff_filter_process_command(AVFilterContext *ctx, const char *cmd, const char *arg, char *res, int res_len, int flags)
Generic processing of user supplied commands that are set in the same way as the filter options...
Definition: avfilter.c:881
void * priv
private data for use by the filter
Definition: avfilter.h:356
#define AVFILTER_FLAG_SLICE_THREADS
The filter supports multithreading by splitting frames into multiple parts and processing them concur...
Definition: avfilter.h:117
#define AV_LOG_DEBUG
Stuff which is only useful for libav* developers.
Definition: log.h:215
const char * arg
Definition: jacosubdec.c:66
simple assert() macros that are a bit more flexible than ISO C assert().
static const AVFilterPad outputs[]
Definition: af_anlmdn.c:401
Context for an Audio FIFO Buffer.
Definition: audio_fifo.c:34
static float distance(float x, float y, int band)
int av_audio_fifo_size(AVAudioFifo *af)
Get the current number of samples in the AVAudioFifo available for reading.
Definition: audio_fifo.c:228
int64_t av_rescale(int64_t a, int64_t b, int64_t c)
Rescale a 64-bit integer with rounding to nearest.
Definition: mathematics.c:129
#define AV_TIME_BASE
Internal time base represented as integer.
Definition: avutil.h:254
#define FFMIN(a, b)
Definition: common.h:105
uint8_t w
Definition: llviddspenc.c:39
float weight_lut[WEIGHT_LUT_SIZE]
Definition: af_anlmdn.c:48
AVFormatContext * ctx
Definition: movenc.c:48
static int query_formats(AVFilterContext *ctx)
Definition: af_anlmdn.c:91
#define s(width, name)
Definition: cbs_vp9.c:257
OutModes
Definition: af_afftdn.c:37
static int filter_channel(AVFilterContext *ctx, void *arg, int ch, int nb_jobs)
Definition: af_anlmdn.c:229
static const AVFilterPad inputs[]
Definition: af_anlmdn.c:392
static int config_filter(AVFilterContext *ctx)
Definition: af_anlmdn.c:149
A list of supported channel layouts.
Definition: formats.h:86
if(ret)
AVFILTER_DEFINE_CLASS(anlmdn)
void(* compute_cache)(float *cache, const float *f, ptrdiff_t S, ptrdiff_t K, ptrdiff_t i, ptrdiff_t jj)
Definition: af_anlmdndsp.h:33
AVSampleFormat
Audio sample formats.
Definition: samplefmt.h:58
static void compute_cache_c(float *cache, const float *f, ptrdiff_t S, ptrdiff_t K, ptrdiff_t i, ptrdiff_t jj)
Definition: af_anlmdn.c:130
static float smooth(DeshakeOpenCLContext *deshake_ctx, float *gauss_kernel, int length, float max_val, AVFifoBuffer *values)
Describe the class of an AVClass context structure.
Definition: log.h:67
Filter definition.
Definition: avfilter.h:145
Rational number (pair of numerator and denominator).
Definition: rational.h:58
const char * name
Filter name.
Definition: avfilter.h:149
#define AVFILTER_FLAG_SUPPORT_TIMELINE_INTERNAL
Same as AVFILTER_FLAG_SUPPORT_TIMELINE_GENERIC, except that the filter will have its filter_frame() c...
Definition: avfilter.h:134
AVFilterLink ** outputs
array of pointers to output links
Definition: avfilter.h:353
enum MovChannelLayoutTag * layouts
Definition: mov_chan.c:434
AVFilterFormats * ff_all_samplerates(void)
Definition: formats.c:422
static int request_frame(AVFilterLink *outlink)
Definition: af_anlmdn.c:342
#define flags(name, subs,...)
Definition: cbs_av1.c:561
AVFilterInternal * internal
An opaque struct for libavfilter internal use.
Definition: avfilter.h:381
int av_audio_fifo_write(AVAudioFifo *af, void **data, int nb_samples)
Write data to an AVAudioFifo.
Definition: audio_fifo.c:112
#define WEIGHT_LUT_SIZE
Definition: af_anlmdn.c:34
The exact code depends on how similar the blocks are and how related they are to the and needs to apply these operations to the correct inlink or outlink if there are several Macros are available to factor that when no extra processing is inlink
AVFrame * cache
Definition: af_anlmdn.c:57
int av_audio_fifo_drain(AVAudioFifo *af, int nb_samples)
Drain data from an AVAudioFifo.
Definition: audio_fifo.c:201
static int filter_frame(AVFilterLink *inlink, AVFrame *in)
Definition: af_anlmdn.c:289
static av_cold void uninit(AVFilterContext *ctx)
Definition: af_anlmdn.c:383
avfilter_execute_func * execute
Definition: internal.h:136
Audio FIFO Buffer.
#define OFFSET(x)
Definition: af_anlmdn.c:74
A list of supported formats for one end of a filter link.
Definition: formats.h:65
AVAudioFifo * fifo
Definition: af_anlmdn.c:61
AudioNLMDNDSPContext dsp
Definition: af_anlmdn.c:64
int av_audio_fifo_peek(AVAudioFifo *af, void **data, int nb_samples)
Peek data from an AVAudioFifo.
Definition: audio_fifo.c:138
An instance of a filter.
Definition: avfilter.h:341
static enum AVSampleFormat sample_fmts[]
Definition: adpcmenc.c:940
FILE * out
Definition: movenc.c:54
int ff_request_frame(AVFilterLink *link)
Request an input frame from the filter at the other end of the link.
Definition: avfilter.c:407
formats
Definition: signature.h:48
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
AVFilterChannelLayouts * ff_all_channel_counts(void)
Construct an AVFilterChannelLayouts coding for any channel layout, with known or unknown disposition...
Definition: formats.c:437
uint8_t ** extended_data
pointers to the data planes/channels.
Definition: frame.h:361
int nb_samples
number of audio samples (per channel) described by this frame
Definition: frame.h:380
for(j=16;j >0;--j)
int ff_set_common_samplerates(AVFilterContext *ctx, AVFilterFormats *samplerates)
Definition: formats.c:576
int i
Definition: input.c:407
#define AV_NOPTS_VALUE
Undefined timestamp value.
Definition: avutil.h:248