[FFmpeg-devel] [PATCH] avformat: add vapoursynth wrapper

James Almer jamrial at gmail.com
Sat Apr 28 04:27:47 EEST 2018


On 4/27/2018 4:37 PM, wm4 wrote:
> From: wm4 <nfxjfg at googlemail.com>
> 
> This can "demux" .vpy files.
> 
> Some minor code copied from other LGPL parts of FFmpeg.
> 
> Possibly support VS compat pixel formats.
> 
> TODO:
> - check whether VS can change format midstream
> - test vfr mode, return proper timestamps when using it
> - drop "lib" prefix?
> ---
>  configure                    |   4 +
>  libavformat/Makefile         |   1 +
>  libavformat/allformats.c     |   1 +
>  libavformat/libvapoursynth.c | 379 +++++++++++++++++++++++++++++++++++++++++++
>  4 files changed, 385 insertions(+)
>  create mode 100644 libavformat/libvapoursynth.c
> 
> diff --git a/configure b/configure
> index 9fa1665496..17e46c5daa 100755
> --- a/configure
> +++ b/configure
> @@ -265,6 +265,7 @@ External library support:
>                             if openssl or gnutls is not used [no]
>    --enable-libtwolame      enable MP2 encoding via libtwolame [no]
>    --enable-libv4l2         enable libv4l2/v4l-utils [no]
> +  --enable-libvapoursynth  enable VapourSynth demuxer [no]
>    --enable-libvidstab      enable video stabilization using vid.stab [no]
>    --enable-libvmaf         enable vmaf filter via libvmaf [no]
>    --enable-libvo-amrwbenc  enable AMR-WB encoding via libvo-amrwbenc [no]
> @@ -1712,6 +1713,7 @@ EXTERNAL_LIBRARY_LIST="
>      libtheora
>      libtwolame
>      libv4l2
> +    libvapoursynth
>      libvorbis
>      libvpx
>      libwavpack
> @@ -3068,6 +3070,7 @@ libspeex_encoder_deps="libspeex"
>  libspeex_encoder_select="audio_frame_queue"
>  libtheora_encoder_deps="libtheora"
>  libtwolame_encoder_deps="libtwolame"
> +libvapoursynth_demuxer_deps="libvapoursynth"
>  libvo_amrwbenc_encoder_deps="libvo_amrwbenc"
>  libvorbis_decoder_deps="libvorbis"
>  libvorbis_encoder_deps="libvorbis libvorbisenc"
> @@ -6041,6 +6044,7 @@ enabled libtwolame        && require libtwolame twolame.h twolame_init -ltwolame
>                               { check_lib libtwolame twolame.h twolame_encode_buffer_float32_interleaved -ltwolame ||
>                                 die "ERROR: libtwolame must be installed and version must be >= 0.3.10"; }
>  enabled libv4l2           && require_pkg_config libv4l2 libv4l2 libv4l2.h v4l2_ioctl
> +enabled libvapoursynth    && require_pkg_config libvapoursynth "vapoursynth >= 42" VapourSynth.h getVapourSynthAPI &&

> require_pkg_config libvapoursynth "vapoursynth-script >= 42" VSScript.h vsscript_init || die "ERROR: vapoursynth or vsscript not found";

die() is needed only with test_pkg_config and check_pkg_config, not
require_pkg_config.

And seeing that vapoursynth-script.pc depends on vapoursynth.pc, you can
simplify all this by only checking for vapoursynth-script.

>  enabled libvidstab        && require_pkg_config libvidstab "vidstab >= 0.98" vid.stab/libvidstab.h vsMotionDetectInit
>  enabled libvmaf           && require_pkg_config libvmaf "libvmaf >= 0.6.2" libvmaf.h compute_vmaf
>  enabled libvo_amrwbenc    && require libvo_amrwbenc vo-amrwbenc/enc_if.h E_IF_init -lvo-amrwbenc
> diff --git a/libavformat/Makefile b/libavformat/Makefile
> index 3eeca5091d..731b7ac714 100644
> --- a/libavformat/Makefile
> +++ b/libavformat/Makefile
> @@ -570,6 +570,7 @@ OBJS-$(CONFIG_LIBRTMPTE_PROTOCOL)        += librtmp.o
>  OBJS-$(CONFIG_LIBSRT_PROTOCOL)           += libsrt.o
>  OBJS-$(CONFIG_LIBSSH_PROTOCOL)           += libssh.o
>  OBJS-$(CONFIG_LIBSMBCLIENT_PROTOCOL)     += libsmbclient.o
> +OBJS-$(CONFIG_LIBVAPOURSYNTH_DEMUXER)    += libvapoursynth.o
>  
>  # protocols I/O
>  OBJS-$(CONFIG_ASYNC_PROTOCOL)            += async.o
> diff --git a/libavformat/allformats.c b/libavformat/allformats.c
> index d582778b3b..67f6c4339c 100644
> --- a/libavformat/allformats.c
> +++ b/libavformat/allformats.c
> @@ -482,6 +482,7 @@ extern AVOutputFormat ff_chromaprint_muxer;
>  extern AVInputFormat  ff_libgme_demuxer;
>  extern AVInputFormat  ff_libmodplug_demuxer;
>  extern AVInputFormat  ff_libopenmpt_demuxer;
> +extern AVInputFormat  ff_libvapoursynth_demuxer;
>  
>  #include "libavformat/muxer_list.c"
>  #include "libavformat/demuxer_list.c"
> diff --git a/libavformat/libvapoursynth.c b/libavformat/libvapoursynth.c
> new file mode 100644
> index 0000000000..95699e81d2
> --- /dev/null
> +++ b/libavformat/libvapoursynth.c
> @@ -0,0 +1,379 @@
> +/*
> + * This file is part of FFmpeg.
> + *
> + * FFmpeg is free software; you can redistribute it and/or
> + * modify it under the terms of the GNU Lesser General Public
> + * License as published by the Free Software Foundation; either
> + * version 2.1 of the License, or (at your option) any later version.
> + *
> + * FFmpeg is distributed in the hope that it will be useful,
> + * but WITHOUT ANY WARRANTY; without even the implied warranty of
> + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
> + * Lesser General Public License for more details.
> + *
> + * You should have received a copy of the GNU Lesser General Public
> + * License along with FFmpeg; if not, write to the Free Software
> + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
> + */
> +
> +/**
> +* @file
> +* VapourSynth demuxer
> +*
> +* Synthesizes vapour (?)
> +*/
> +
> +#include <VapourSynth.h>
> +#include <VSScript.h>
> +
> +#include "libavutil/avassert.h"
> +#include "libavutil/avstring.h"
> +#include "libavutil/eval.h"
> +#include "libavutil/imgutils.h"
> +#include "libavutil/opt.h"
> +#include "libavutil/pixdesc.h"
> +#include "avformat.h"
> +#include "internal.h"
> +
> +typedef struct VSContext {
> +    const AVClass *class;
> +
> +    const VSAPI *vsapi;
> +    VSCore *vscore;
> +    VSScript *vss;
> +
> +    VSNodeRef *outnode;
> +    int is_cfr;
> +    int current_frame;
> +
> +    int c_order[4];
> +
> +    /* options */
> +    int64_t max_size;
> +} VSContext;
> +
> +#define OFFSET(x) offsetof(VSContext, x)
> +#define A AV_OPT_FLAG_AUDIO_PARAM
> +#define D AV_OPT_FLAG_DECODING_PARAM
> +static const AVOption options[] = {
> +    {"max_size",    "set max file size supported (in bytes)", OFFSET(max_size),    AV_OPT_TYPE_INT64, {.i64 = 1 * 1024 * 1024}, 0,    SIZE_MAX - 1, A|D},
> +    {NULL}
> +};
> +
> +static int read_close_vs(AVFormatContext *s)
> +{
> +    VSContext *vs = s->priv_data;
> +
> +    if (vs->outnode)
> +        vs->vsapi->freeNode(vs->outnode);
> +
> +    vsscript_freeScript(vs->vss);
> +    vs->vss = NULL;
> +    vs->vsapi = NULL;
> +    vs->vscore = NULL;
> +    vs->outnode = NULL;
> +
> +    vsscript_finalize();
> +
> +    return 0;
> +}
> +
> +static int is_native_endian(enum AVPixelFormat pixfmt)
> +{
> +    enum AVPixelFormat other = av_pix_fmt_swap_endianness(pixfmt);
> +    const AVPixFmtDescriptor *pd;
> +    if (other == AV_PIX_FMT_NONE || other == pixfmt)
> +        return 1; // not affected by byte order
> +    pd = av_pix_fmt_desc_get(pixfmt);
> +    return pd && (!!HAVE_BIGENDIAN == !!(pd->flags & AV_PIX_FMT_FLAG_BE));
> +}
> +
> +static enum AVPixelFormat match_pixfmt(const VSFormat *vsf, int c_order[4])
> +{
> +    static const int yuv_order[4] = {0, 1, 2, 0};
> +    static const int rgb_order[4] = {1, 2, 0, 0};
> +    const AVPixFmtDescriptor *pd;
> +
> +    for (pd = av_pix_fmt_desc_next(NULL); pd; pd = av_pix_fmt_desc_next(pd)) {
> +        int is_rgb, is_yuv, i, *order;
> +        enum AVPixelFormat pixfmt;
> +
> +        pixfmt = av_pix_fmt_desc_get_id(pd);
> +
> +        if (pd->flags & (AV_PIX_FMT_FLAG_BAYER | AV_PIX_FMT_FLAG_ALPHA |
> +                         AV_PIX_FMT_FLAG_HWACCEL | AV_PIX_FMT_FLAG_BITSTREAM))
> +            continue;
> +
> +        if (pd->log2_chroma_w != vsf->subSamplingW ||
> +            pd->log2_chroma_h != vsf->subSamplingH)
> +            continue;
> +
> +        is_rgb = vsf->colorFamily == cmRGB;
> +        if (is_rgb != !!(pd->flags & AV_PIX_FMT_FLAG_RGB))
> +            continue;
> +
> +        is_yuv = vsf->colorFamily == cmYUV ||
> +                 vsf->colorFamily == cmYCoCg ||
> +                 vsf->colorFamily == cmGray;
> +        if (!is_rgb && !is_yuv)
> +            continue;
> +
> +        if (vsf->sampleType != ((pd->flags & AV_PIX_FMT_FLAG_FLOAT) ? stFloat : stInteger))
> +            continue;
> +
> +        if (av_pix_fmt_count_planes(pixfmt) != vsf->numPlanes)
> +            continue;
> +
> +        if (strncmp(pd->name, "xyz", 3) == 0)
> +            continue;
> +
> +        if (!is_native_endian(pixfmt))
> +            continue;
> +
> +        order = is_yuv ? yuv_order : rgb_order;
> +
> +        for (i = 0; i < pd->nb_components; i++) {
> +            const AVComponentDescriptor *c = &pd->comp[i];
> +            if (order[c->plane] != i ||
> +                c->offset != 0 || c->shift != 0 ||
> +                c->step != vsf->bytesPerSample ||
> +                c->depth != vsf->bitsPerSample)
> +                goto cont;
> +        }
> +
> +        // Use it.
> +        memcpy(c_order, order, sizeof(int[4]));
> +        return pixfmt;
> +
> +    cont: ;
> +    }
> +
> +    return AV_PIX_FMT_NONE;
> +}
> +
> +static int read_header_vs(AVFormatContext *s)
> +{
> +    AVStream *st;
> +    AVIOContext *pb = s->pb;
> +    VSContext *vs = s->priv_data;
> +    int64_t sz = avio_size(pb);
> +    char *buf = NULL;
> +    char dummy;
> +    const VSVideoInfo *info;
> +    int err;
> +
> +    vsscript_init();
> +
> +    if (sz < 0 || sz > vs->max_size) {
> +        if (sz < 0)
> +            av_log(s, AV_LOG_WARNING, "Could not determine file size\n");
> +        sz = vs->max_size;
> +    }
> +
> +    buf = av_malloc(sz + 1);
> +    if (!buf) {
> +        err = AVERROR(ENOMEM);
> +        goto done;
> +    }
> +    sz = avio_read(pb, buf, sz);
> +
> +    if (sz < 0) {
> +        av_log(s, AV_LOG_ERROR, "Could not read script.\n");
> +        err = sz;
> +        goto done;
> +    }
> +
> +    // Data left means our buffer (the max_size option) is too small
> +    if (avio_read(pb, &dummy, 1) == 1) {
> +        av_log(s, AV_LOG_ERROR, "File size is larger than max_size option "
> +               "value %"PRIi64", consider increasing the max_size option\n",
> +               vs->max_size);
> +        err = AVERROR_BUFFER_TOO_SMALL;
> +        goto done;
> +    }
> +
> +    if (vsscript_createScript(&vs->vss)) {
> +        av_log(s, AV_LOG_ERROR, "Failed to create script instance.\n");
> +        err = AVERROR_EXTERNAL;
> +        goto done;
> +    }
> +
> +    buf[sz] = '\0';
> +    if (vsscript_evaluateScript(&vs->vss, buf, s->url, 0)) {
> +        const char *msg = vsscript_getError(vs->vss);
> +        av_log(s, AV_LOG_ERROR, "Failed to parse script: %s\n", msg ? msg : "(unknown)");
> +        err = AVERROR_EXTERNAL;
> +        goto done;
> +    }
> +
> +    vs->vsapi = vsscript_getVSApi();
> +    vs->vscore = vsscript_getCore(vs->vss);
> +
> +    vs->outnode = vsscript_getOutput(vs->vss, 0);
> +    if (!vs->outnode) {
> +        av_log(s, AV_LOG_ERROR, "Could not get script output node.\n");
> +        err = AVERROR_EXTERNAL;
> +        goto done;
> +    }
> +
> +    st = avformat_new_stream(s, NULL);
> +    if (!st) {
> +        err = AVERROR(ENOMEM);
> +        goto done;
> +    }
> +
> +    info = vs->vsapi->getVideoInfo(vs->outnode);
> +
> +    if (info->fpsDen) {
> +        vs->is_cfr = 1;
> +        avpriv_set_pts_info(st, 64, info->fpsDen, info->fpsNum);
> +        st->duration = info->numFrames;
> +    } else {
> +        // VFR. Just set "something".
> +        avpriv_set_pts_info(st, 64, 1, AV_TIME_BASE);
> +        s->ctx_flags |= AVFMTCTX_UNSEEKABLE;
> +    }
> +
> +    st->codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
> +    st->codecpar->codec_id = AV_CODEC_ID_WRAPPED_AVFRAME;

What exactly is the use case for this?

> +    st->codecpar->width = info->width;
> +    st->codecpar->height = info->height;
> +    st->codecpar->format = match_pixfmt(info->format, vs->c_order);
> +
> +    if (st->codecpar->format == AV_PIX_FMT_NONE) {
> +        av_log(s, AV_LOG_ERROR, "Unsupported VS pixel format %s\n", info->format->name);
> +        err = AVERROR_EXTERNAL;
> +        goto done;
> +    }
> +    av_log(s, AV_LOG_VERBOSE, "VS format %s -> pixfmt %s\n", info->format->name,
> +           av_get_pix_fmt_name(st->codecpar->format));
> +
> +    if (info->format->colorFamily == cmYCoCg)
> +        st->codecpar->color_space = AVCOL_SPC_YCGCO;
> +
> +done:
> +    av_free(buf);
> +    if (err < 0)
> +        read_close_vs(s);
> +    return err;
> +}
> +
> +static void free_frame(void *opaque, uint8_t *data)
> +{
> +    AVFrame *frame = (AVFrame *)data;
> +
> +    av_frame_free(&frame);
> +}
> +
> +static int read_packet_vs(AVFormatContext *s, AVPacket *pkt)
> +{
> +    VSContext *vs = s->priv_data;
> +    AVStream *st = s->streams[0];
> +    AVFrame *frame = NULL;
> +    char vserr[80];
> +    const VSFrameRef *vsframe = NULL;
> +    const VSVideoInfo *info = vs->vsapi->getVideoInfo(vs->outnode);
> +    int err = 0;
> +    const uint8_t *src_data[4];
> +    int src_linesizes[4];
> +    int i;
> +
> +    if (vs->current_frame >= info->numFrames)
> +        return AVERROR_EOF;
> +
> +    vsframe = vs->vsapi->getFrame(vs->current_frame, vs->outnode, vserr, sizeof(vserr));
> +    if (!vsframe) {
> +        av_log(s, AV_LOG_ERROR, "Error getting frame: %s\n", vserr);
> +        err = AVERROR_EXTERNAL;
> +        goto end;
> +    }
> +
> +    frame = av_frame_alloc();
> +    if (!frame) {
> +        err = AVERROR(ENOMEM);
> +        goto end;
> +    }
> +
> +    frame->format       = st->codecpar->format;
> +    frame->width        = st->codecpar->width;
> +    frame->height       = st->codecpar->height;
> +    frame->colorspace   = st->codecpar->color_space;
> +
> +    av_assert0(vs->vsapi->getFrameWidth(vsframe, 0) == frame->width);
> +    av_assert0(vs->vsapi->getFrameHeight(vsframe, 0) == frame->height);
> +
> +    err = av_frame_get_buffer(frame, 0);
> +    if (err < 0)
> +        goto end;
> +
> +    for (i = 0; i < info->format->numPlanes; i++) {
> +        int p = vs->c_order[i];
> +        src_data[i] = vs->vsapi->getReadPtr(vsframe, p);
> +        src_linesizes[i] = vs->vsapi->getStride(vsframe, p);
> +    }
> +
> +    av_image_copy(frame->data, frame->linesize, src_data, src_linesizes,
> +                  frame->format, frame->width, frame->height);
> +
> +    pkt->buf = av_buffer_create((uint8_t*)frame, sizeof(*frame),

So, the whole wrapped avframe is pretty much fucked up since conception.
Did nobody notice that lavc/wrapped_avframe.c is using sizeof(AVFrame)?

Much like in here, it's wrong and Bad Things(tm) will happen as soon as
we add a new field.

> +                                free_frame, NULL, 0);
> +    if (!pkt->buf) {
> +        err = AVERROR(ENOMEM);
> +        goto end;
> +    }
> +
> +    frame = NULL; // pkt owns it now
> +
> +    pkt->data   = pkt->buf->data;
> +    pkt->size   = pkt->buf->size;
> +    pkt->flags |= AV_PKT_FLAG_TRUSTED;
> +
> +    if (vs->is_cfr)
> +        pkt->pts = vs->current_frame;
> +
> +    vs->current_frame++;
> +
> +end:
> +    if (err < 0)
> +        av_packet_unref(pkt);
> +    av_frame_free(&frame);
> +    vs->vsapi->freeFrame(vsframe);
> +    return err;
> +}
> +
> +static int read_seek_vs(AVFormatContext *s, int stream_idx, int64_t ts, int flags)
> +{
> +    VSContext *vs = s->priv_data;
> +
> +    if (!vs->is_cfr)
> +        return AVERROR(ENOSYS);
> +
> +    vs->current_frame = FFMIN(FFMAX(0, ts), s->streams[0]->duration);
> +    return 0;
> +}
> +
> +static int probe_vs(AVProbeData *p)
> +{
> +    // Explicitly do not support this. VS scripts are written in Python, and
> +    // can run arbitrary code on the user's system.
> +    return 0;
> +}
> +
> +static const AVClass class_vs = {
> +    .class_name = "VapourSynth demuxer",
> +    .item_name  = av_default_item_name,
> +    .option     = options,
> +    .version    = LIBAVUTIL_VERSION_INT,
> +};
> +
> +AVInputFormat ff_libvapoursynth_demuxer = {
> +    .name           = "libvapoursynth",
> +    .long_name      = NULL_IF_CONFIG_SMALL("VapourSynth demuxer"),
> +    .priv_data_size = sizeof(VSContext),
> +    .read_probe     = probe_vs,
> +    .read_header    = read_header_vs,
> +    .read_packet    = read_packet_vs,
> +    .read_close     = read_close_vs,
> +    .read_seek      = read_seek_vs,
> +    .priv_class     = &class_vs,
> +};
> 



More information about the ffmpeg-devel mailing list