movenc_ttml.c

Go to the documentation of this file.

/*
 * MP4, ISMV Muxer TTML helpers
 * Copyright (c) 2021 24i
 *
 * This file is part of FFmpeg.
 *
 * FFmpeg is free software; you can redistribute it and/or
 * modify it under the terms of the GNU Lesser General Public
 * License as published by the Free Software Foundation; either
 * version 2.1 of the License, or (at your option) any later version.
 *
 * FFmpeg is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public
 * License along with FFmpeg; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 */
 
#include "libavutil/mem.h"
#include "avformat.h"
#include "avio_internal.h"
#include "isom.h"
#include "movenc.h"
#include "movenc_ttml.h"
#include "packet_internal.h"
 
static const unsigned char empty_ttml_document[] =
    "<tt xml:lang=\"\" xmlns=\"http://www.w3.org/ns/ttml\" />";
 
static int mov_init_ttml_writer(MOVTrack *track, AVFormatContext **out_ctx)
{
    AVStream *movenc_stream = track->st, *ttml_stream = NULL;
    int ret = AVERROR_BUG;
 
    if ((ret = avformat_alloc_output_context2(out_ctx, NULL,
                                              "ttml", NULL)) < 0)
        return ret;
 
    if ((ret = avio_open_dyn_buf(&(*out_ctx)->pb)) < 0)
        return ret;
 
    if (!(ttml_stream = avformat_new_stream(*out_ctx, NULL))) {
        return AVERROR(ENOMEM);
    }
 
    if ((ret = avcodec_parameters_copy(ttml_stream->codecpar,
                                       movenc_stream->codecpar)) < 0)
        return ret;
 
    ttml_stream->time_base = movenc_stream->time_base;
 
    return 0;
}
 
static void mov_calculate_start_and_end_of_other_tracks(
    AVFormatContext *s, MOVTrack *track, int64_t *start_pts, int64_t *end_pts)
{
    MOVMuxContext *mov = s->priv_data;
 
    // Initialize at the end of the previous document/fragment, which is NOPTS
    // until the first fragment is created.
    int64_t max_track_end_dts = *start_pts = track->end_pts;
 
    for (unsigned int i = 0; i < s->nb_streams; i++) {
        MOVTrack *other_track = &mov->tracks[i];
 
        // Skip our own track, any other track that needs squashing,
        // or any track which still has its start_dts at NOPTS or
        // any track that did not yet get any packets.
        if (track == other_track ||
            other_track->squash_fragment_samples_to_one ||
            other_track->start_dts == AV_NOPTS_VALUE ||
            !other_track->entry) {
            continue;
        }
 
        int64_t picked_start = av_rescale_q_rnd(other_track->cluster[0].dts + other_track->cluster[0].cts,
                                                other_track->st->time_base,
                                                track->st->time_base,
                                                AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX);
        int64_t picked_end   = av_rescale_q_rnd(other_track->end_pts,
                                                other_track->st->time_base,
                                                track->st->time_base,
                                                AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX);
 
        if (*start_pts == AV_NOPTS_VALUE)
            *start_pts = picked_start;
        else if (picked_start >= track->end_pts)
            *start_pts = FFMIN(*start_pts, picked_start);
 
        max_track_end_dts = FFMAX(max_track_end_dts, picked_end);
    }
 
    *end_pts = max_track_end_dts;
}
 
static int mov_write_ttml_document_from_queue(AVFormatContext *s,
                                              AVFormatContext *ttml_ctx,
                                              MOVTrack *track,
                                              AVPacket *pkt,
                                              int64_t *out_start_ts,
                                              int64_t *out_duration)
{
    int ret = AVERROR_BUG;
    int64_t start_ts = track->start_dts == AV_NOPTS_VALUE ?
                       0 : (track->start_dts + track->track_duration);
    int64_t end_ts   = start_ts;
    unsigned int time_limited = 0;
    PacketList back_to_queue_list = { 0 };
 
    if (*out_start_ts != AV_NOPTS_VALUE) {
        // we have non-nopts values here, thus we have been given a time range
        time_limited = 1;
        start_ts = *out_start_ts;
        end_ts   = *out_start_ts + *out_duration;
    }
 
    if ((ret = avformat_write_header(ttml_ctx, NULL)) < 0) {
        return ret;
    }
 
    while (!ff_packet_list_get(&track->squashed_packet_queue, pkt)) {
        int64_t pts_before      = pkt->pts;
        int64_t duration_before = pkt->duration;
 
        if (time_limited) {
            // special cases first:
            if (pkt->pts + pkt->duration < start_ts) {
                // too late for our fragment, unfortunately
                // unref and proceed to next packet in queue.
                av_log(s, AV_LOG_WARNING,
                       "Very late TTML packet in queue, dropping packet with "
                       "pts: %"PRId64", duration: %"PRId64"\n",
                       pkt->pts, pkt->duration);
                av_packet_unref(pkt);
                continue;
            } else if (pkt->pts >= end_ts) {
                // starts after this fragment, put back to original queue
                ret = ff_packet_list_put(&track->squashed_packet_queue,
                                             pkt, NULL,
                                             FF_PACKETLIST_FLAG_PREPEND);
                if (ret < 0)
                    goto cleanup;
 
                break;
            }
 
            // limit packet pts to start_ts
            if (pkt->pts < start_ts) {
                pkt->duration -= start_ts - pkt->pts;
                pkt->pts = start_ts;
            }
 
            if (pkt->pts + pkt->duration > end_ts) {
                // goes over our current fragment, create duplicate and
                // put it back to list after iteration has finished in
                // order to handle multiple subtitles at the same time.
                int64_t offset = end_ts - pkt->pts;
 
                ret = ff_packet_list_put(&back_to_queue_list,
                                             pkt, av_packet_ref,
                                             FF_PACKETLIST_FLAG_PREPEND);
                if (ret < 0)
                    goto cleanup;
 
                back_to_queue_list.head->pkt.pts =
                back_to_queue_list.head->pkt.dts =
                back_to_queue_list.head->pkt.pts + offset;
                back_to_queue_list.head->pkt.duration -= offset;
 
                // and for our normal packet we just set duration to offset
                pkt->duration = offset;
            }
        } else {
            end_ts = FFMAX(end_ts, pkt->pts + pkt->duration);
        }
 
        av_log(s, AV_LOG_TRACE,
               "TTML packet writeout: pts: %"PRId64" (%"PRId64"), "
               "duration: %"PRId64"\n",
               pkt->pts, pkt->pts - start_ts, pkt->duration);
        if (pkt->pts != pts_before || pkt->duration != duration_before) {
            av_log(s, AV_LOG_TRACE,
                   "Adjustments: pts: %"PRId64", duration: %"PRId64"\n",
                   pkt->pts - pts_before, pkt->duration - duration_before);
        }
 
        // in case of the 'dfxp' muxing mode, each written document is offset
        // to its containing sample's beginning.
        if (track->par->codec_tag == MOV_ISMV_TTML_TAG) {
            pkt->dts = pkt->pts = (pkt->pts - start_ts);
        }
 
        pkt->stream_index = 0;
 
        av_packet_rescale_ts(pkt, track->st->time_base,
                             ttml_ctx->streams[pkt->stream_index]->time_base);
 
        if ((ret = av_write_frame(ttml_ctx, pkt)) < 0) {
            goto cleanup;
        }
 
        av_packet_unref(pkt);
    }
 
    if ((ret = av_write_trailer(ttml_ctx)) < 0)
        goto cleanup;
 
    *out_start_ts = start_ts;
    *out_duration = end_ts - start_ts;
 
    ret = 0;
 
cleanup:
    av_packet_unref(pkt);
    while (!ff_packet_list_get(&back_to_queue_list, pkt)) {
        ret = ff_packet_list_put(&track->squashed_packet_queue,
                                     pkt, av_packet_ref,
                                     FF_PACKETLIST_FLAG_PREPEND);
 
        // unrelated to whether we succeed or not, we unref the packet
        // received from the temporary list.
        av_packet_unref(pkt);
 
        if (ret < 0) {
            ff_packet_list_free(&back_to_queue_list);
            break;
        }
    }
    return ret;
}
 
int ff_mov_generate_squashed_ttml_packet(AVFormatContext *s,
                                         MOVTrack *track, AVPacket *pkt)
{
    MOVMuxContext *mov = s->priv_data;
    AVFormatContext *ttml_ctx = NULL;
    // values for the generated AVPacket
    int64_t start_ts = AV_NOPTS_VALUE;
    int64_t duration = 0;
 
    int ret = AVERROR_BUG;
 
    if ((ret = mov_init_ttml_writer(track, &ttml_ctx)) < 0) {
        av_log(s, AV_LOG_ERROR, "Failed to initialize the TTML writer: %s\n",
               av_err2str(ret));
        goto cleanup;
    }
 
    if (mov->flags & FF_MOV_FLAG_FRAGMENT) {
        int64_t calculated_start = AV_NOPTS_VALUE;
        int64_t calculated_end = AV_NOPTS_VALUE;
 
        mov_calculate_start_and_end_of_other_tracks(s, track, &calculated_start, &calculated_end);
 
        if (calculated_start != AV_NOPTS_VALUE) {
            start_ts = calculated_start;
            duration = calculated_end - calculated_start;
            av_log(s, AV_LOG_VERBOSE,
                   "Calculated subtitle fragment start: %"PRId64", "
                   "duration: %"PRId64"\n",
                   start_ts, duration);
        }
    }
 
    if (!track->squashed_packet_queue.head) {
        // empty queue, write minimal empty document with zero duration
        avio_write(ttml_ctx->pb, empty_ttml_document,
                   sizeof(empty_ttml_document) - 1);
        if (start_ts == AV_NOPTS_VALUE) {
            start_ts = 0;
            duration = 0;
        }
        goto generate_packet;
    }
 
    if ((ret = mov_write_ttml_document_from_queue(s, ttml_ctx, track, pkt,
                                                  &start_ts,
                                                  &duration)) < 0) {
        av_log(s, AV_LOG_ERROR,
               "Failed to generate a squashed TTML packet from the packet "
               "queue: %s\n",
               av_err2str(ret));
        goto cleanup;
    }
 
generate_packet:
    {
        // Generate an AVPacket from the data written into the dynamic buffer.
        uint8_t *buf = NULL;
        int buf_len = avio_close_dyn_buf(ttml_ctx->pb, &buf);
        ttml_ctx->pb = NULL;
 
        if ((ret = av_packet_from_data(pkt, buf, buf_len)) < 0) {
            av_log(s, AV_LOG_ERROR,
                   "Failed to create a TTML AVPacket from AVIO data: %s\n",
                   av_err2str(ret));
            av_freep(&buf);
            goto cleanup;
        }
 
        pkt->pts = pkt->dts = start_ts;
        pkt->duration = duration;
        pkt->flags |= AV_PKT_FLAG_KEY;
    }
 
    ret = 0;
 
cleanup:
    if (ttml_ctx)
        ffio_free_dyn_buf(&ttml_ctx->pb);
 
    avformat_free_context(ttml_ctx);
    return ret;
}