Skip to content

Commit

Permalink
Demuxer reopening fixes
Browse files Browse the repository at this point in the history 
This is quite deep and important change. Thing is, old code had
"demuxer reuse" attempt. And unfortunately it neither made sense
(next to nothing performance improvement, complex code, limited
to MPEGTS streams), nor was working fine.

During Low Latency work I tried to eliminate demuxer reusing
because I wanted single demuxer logic to make input I/O plug-in
easier. It turned out that so modified code failed certain tests
such as Nvidia_AudioOnly.

It took me a great deal of debugging, but I found out that there
is bug in demuxer reuse procedure: namely only first muxer open
attempt _really_ checks what is in the segment provided, the rest
just assumes that same streams are there.

So basically, in the series of "audio/video"/"audio without video"
/"audio/video" segments all will be perceived as containing video!
This "patched over" *AudioOnly tests, and let them run, but only
because Transcoder "though" that it has video all the time.

And this simply wasn't true. When I removed "muxer reusing" code,
the Nvidia-dedicated "preserve video decoder" tactics was falling
over the fact that pixel format changed (because it changed from
AV_PIX_FMT_SOMETHING into AV_PIX_FMT_NONE) and tried to create
hardware video decoder for AV_PIX_FMT_NONE, and failed.

Fixing all that finally allowed me to get rid of recursive call to
open_input when HW decoder needs to be reset. All is well that
ends well, I guess...
  • Loading branch information
@MikeIndiaAlpha
MikeIndiaAlpha committed Jul 22, 2022
1 parent 730cbec commit 9196c7c
Show file tree
Hide file tree
Showing 6 changed files with 119 additions and 158 deletions.
193 changes: 91 additions & 102 deletions ffmpeg/decoder.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,29 +128,36 @@ static enum AVPixelFormat get_hw_pixfmt(AVCodecContext *vc, const enum AVPixelFo
}


int open_audio_decoder(struct input_ctx *ctx, AVCodec *codec)
static int open_audio_decoder(struct input_ctx *ctx, AVCodec *codec)
{
int ret = 0;
AVFormatContext *ic = ctx->ic;

// open audio decoder
AVCodecContext * ac = avcodec_alloc_context3(codec);
if (!ac) LPMS_ERR(open_audio_err, "Unable to alloc audio codec");
if (ctx->ac) LPMS_WARN("An audio context was already open!");
ctx->ac = ac;
ret = avcodec_parameters_to_context(ac, ic->streams[ctx->ai]->codecpar);
if (ret < 0) LPMS_ERR(open_audio_err, "Unable to assign audio params");
ret = avcodec_open2(ac, codec, NULL);
if (ret < 0) LPMS_ERR(open_audio_err, "Unable to open audio decoder");

AVCodecContext * ac = avcodec_alloc_context3(codec);
if (!ac) LPMS_ERR(open_audio_err, "Unable to alloc audio codec");
if (ctx->ac) LPMS_WARN("An audio context was already open!");
ctx->ac = ac;
ret = avcodec_parameters_to_context(ac, ic->streams[ctx->ai]->codecpar);
if (ret < 0) LPMS_ERR(open_audio_err, "Unable to assign audio params");
ret = avcodec_open2(ac, codec, NULL);
if (ret < 0) LPMS_ERR(open_audio_err, "Unable to open audio decoder");
ctx->last_frame_a = av_frame_alloc();
if (!ctx->last_frame_a) LPMS_ERR(open_audio_err, "Unable to alloc last_frame_a");
return 0;

open_audio_err:
free_input(ctx, FORCE_CLOSE_HW_DECODER);
return ret;
}

char* get_hw_decoder(int ff_codec_id, int hw_type)
static void close_audio_decoder(struct input_ctx *ictx)
{
if (ictx->ac) avcodec_free_context(&ictx->ac);
if (ictx->last_frame_a) av_frame_free(&ictx->last_frame_a);
}

static char* get_hw_decoder(int ff_codec_id, int hw_type)
{
switch (hw_type) {
case AV_HWDEVICE_TYPE_CUDA:
Expand Down Expand Up @@ -184,47 +191,49 @@ char* get_hw_decoder(int ff_codec_id, int hw_type)
}
}

int open_video_decoder(struct input_ctx *ctx, AVCodec *codec)
static int open_video_decoder(struct input_ctx *ctx, AVCodec *codec)
{
int ret = 0;
AVDictionary **opts = NULL;
AVFormatContext *ic = ctx->ic;
// open video decoder
if (ctx->hw_type > AV_HWDEVICE_TYPE_NONE) {
char* decoder_name = get_hw_decoder(codec->id, ctx->hw_type);
if (!*decoder_name) {
ret = lpms_ERR_INPUT_CODEC;
LPMS_ERR(open_decoder_err, "Input codec does not support hardware acceleration");
}
AVCodec *c = avcodec_find_decoder_by_name(decoder_name);
if (c) codec = c;
else LPMS_WARN("Nvidia decoder not found; defaulting to software");
if (AV_PIX_FMT_YUV420P != ic->streams[ctx->vi]->codecpar->format &&
AV_PIX_FMT_YUVJ420P != ic->streams[ctx->vi]->codecpar->format) {
// TODO check whether the color range is truncated if yuvj420p is used
ret = lpms_ERR_INPUT_PIXFMT;
LPMS_ERR(open_decoder_err, "Non 4:2:0 pixel format detected in input");
}
if (ctx->hw_type > AV_HWDEVICE_TYPE_NONE) {
char* decoder_name = get_hw_decoder(codec->id, ctx->hw_type);
if (!*decoder_name) {
ret = lpms_ERR_INPUT_CODEC;
LPMS_ERR(open_decoder_err, "Input codec does not support hardware acceleration");
}
AVCodecContext *vc = avcodec_alloc_context3(codec);
if (!vc) LPMS_ERR(open_decoder_err, "Unable to alloc video codec");
ctx->vc = vc;
ret = avcodec_parameters_to_context(vc, ic->streams[ctx->vi]->codecpar);
if (ret < 0) LPMS_ERR(open_decoder_err, "Unable to assign video params");
vc->opaque = (void*)ctx;
// XXX Could this break if the original device falls out of scope in golang?
if (ctx->hw_type == AV_HWDEVICE_TYPE_CUDA) {
// First set the hw device then set the hw frame
ret = av_hwdevice_ctx_create(&ctx->hw_device_ctx, ctx->hw_type, ctx->device, NULL, 0);
if (ret < 0) LPMS_ERR(open_decoder_err, "Unable to open hardware context for decoding")
vc->hw_device_ctx = av_buffer_ref(ctx->hw_device_ctx);
vc->get_format = get_hw_pixfmt;
AVCodec *c = avcodec_find_decoder_by_name(decoder_name);
if (c) codec = c;
else LPMS_WARN("Nvidia decoder not found; defaulting to software");
if (AV_PIX_FMT_YUV420P != ic->streams[ctx->vi]->codecpar->format &&
AV_PIX_FMT_YUVJ420P != ic->streams[ctx->vi]->codecpar->format) {
// TODO check whether the color range is truncated if yuvj420p is used
ret = lpms_ERR_INPUT_PIXFMT;
LPMS_ERR(open_decoder_err, "Non 4:2:0 pixel format detected in input");
}
vc->pkt_timebase = ic->streams[ctx->vi]->time_base;
av_opt_set(vc->priv_data, "xcoder-params", ctx->xcoderParams, 0);
ret = avcodec_open2(vc, codec, opts);
if (ret < 0) LPMS_ERR(open_decoder_err, "Unable to open video decoder");
}

AVCodecContext *vc = avcodec_alloc_context3(codec);
if (!vc) LPMS_ERR(open_decoder_err, "Unable to alloc video codec");
ctx->vc = vc;
ret = avcodec_parameters_to_context(vc, ic->streams[ctx->vi]->codecpar);
if (ret < 0) LPMS_ERR(open_decoder_err, "Unable to assign video params");
vc->opaque = (void*)ctx;
// XXX Could this break if the original device falls out of scope in golang?
if (ctx->hw_type == AV_HWDEVICE_TYPE_CUDA) {
// First set the hw device then set the hw frame
ret = av_hwdevice_ctx_create(&ctx->hw_device_ctx, ctx->hw_type, ctx->device, NULL, 0);
if (ret < 0) LPMS_ERR(open_decoder_err, "Unable to open hardware context for decoding")
vc->hw_device_ctx = av_buffer_ref(ctx->hw_device_ctx);
vc->get_format = get_hw_pixfmt;
}
vc->pkt_timebase = ic->streams[ctx->vi]->time_base;
av_opt_set(vc->priv_data, "xcoder-params", ctx->xcoderParams, 0);
ret = avcodec_open2(vc, codec, opts);
if (ret < 0) LPMS_ERR(open_decoder_err, "Unable to open video decoder");
ctx->last_frame_v = av_frame_alloc();
if (!ctx->last_frame_v) LPMS_ERR(open_decoder_err, "Unable to alloc last_frame_v");
return 0;

open_decoder_err:
Expand All @@ -233,6 +242,16 @@ int open_video_decoder(struct input_ctx *ctx, AVCodec *codec)
return ret;
}

static void close_video_decoder(struct input_ctx *ictx)
{
if (ictx->vc) {
if (ictx->vc->hw_device_ctx) av_buffer_unref(&ictx->vc->hw_device_ctx);
avcodec_free_context(&ictx->vc);
}
if (ictx->hw_device_ctx) av_buffer_unref(&ictx->hw_device_ctx);
if (ictx->last_frame_v) av_frame_free(&ictx->last_frame_v);
}

int open_input(input_params *params, struct input_ctx *ctx)
{
char *inp = params->fname;
Expand All @@ -247,52 +266,47 @@ int open_input(input_params *params, struct input_ctx *ctx)
ctx->device = params->device;

// open demuxer
if (!ctx->ic) {
ret = avformat_open_input(&ctx->ic, inp, NULL, NULL);
if (ret < 0) LPMS_ERR(open_input_err, "demuxer: Unable to open input");
ret = avformat_find_stream_info(ctx->ic, NULL);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to find input info");
} else if (!ctx->ic->pb) {
// reopen input segment file IO context if needed
ret = avio_open(&ctx->ic->pb, inp, AVIO_FLAG_READ);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to reopen file");
} else reopen_decoders = 0;
ret = avformat_open_input(&ctx->ic, inp, NULL, NULL);
if (ret < 0) LPMS_ERR(open_input_err, "demuxer: Unable to open input");
ret = avformat_find_stream_info(ctx->ic, NULL);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to find input info");

AVCodec *video_codec = NULL;
AVCodec *audio_codec = NULL;
ctx->vi = av_find_best_stream(ctx->ic, AVMEDIA_TYPE_VIDEO, -1, -1, &video_codec, 0);
ctx->ai = av_find_best_stream(ctx->ic, AVMEDIA_TYPE_AUDIO, -1, -1, &audio_codec, 0);

if (AV_HWDEVICE_TYPE_CUDA == ctx->hw_type && ctx->vi >= 0) {
if (ctx->last_format == AV_PIX_FMT_NONE) ctx->last_format = ctx->ic->streams[ctx->vi]->codecpar->format;
else if (ctx->ic->streams[ctx->vi]->codecpar->format != ctx->last_format) {
// Now be careful here. It appears that in certain situation (such as .ts
// stream without video stream) ctx->vi will be set to 0, but the format will
// be set to AV_PIX_FMT_NONE and both width and height will be zero, etc
// This is normally fine, but when re-using video decoder we have to be
// extra careful
enum AVPixelFormat format = ctx->ic->streams[ctx->vi]->codecpar->format;
if ((AV_HWDEVICE_TYPE_CUDA == ctx->hw_type) && (ctx->vi >= 0)
&& (AV_PIX_FMT_NONE != format)) {
if (ctx->last_format == AV_PIX_FMT_NONE) ctx->last_format = format;
else if (format != ctx->last_format) {
LPMS_WARN("Input pixel format has been changed in the middle.");
ctx->last_format = ctx->ic->streams[ctx->vi]->codecpar->format;
ctx->last_format = format;
// if the decoder is not re-opened when the video pixel format is changed,
// the decoder tries HW decoding with the video context initialized to a pixel format different from the input one.
// to handle a change in the input pixel format,
// we close the demuxer and re-open the decoder by calling open_input().
free_input(ctx, FORCE_CLOSE_HW_DECODER);
ret = open_input(params, ctx);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to reopen video demuxer for HW decoding");
reopen_decoders = 0;
// we close the decoder so it will get reopened later
close_video_decoder(ctx);
}
}

if (reopen_decoders) {
if (!ctx->dv && (ctx->vi >= 0) &&
(!ctx->vc || (ctx->hw_type == AV_HWDEVICE_TYPE_NONE))) {
ret = open_video_decoder(ctx, video_codec);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to open video decoder")
ctx->last_frame_v = av_frame_alloc();
if (!ctx->last_frame_v) LPMS_ERR(open_input_err, "Unable to alloc last_frame_v");
if (!ctx->dv && (ctx->vi >= 0) && (AV_PIX_FMT_NONE != format)) {
if (!ctx->vc || (ctx->hw_type == AV_HWDEVICE_TYPE_NONE)) {
ret = open_video_decoder(ctx, video_codec);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to open video decoder")
}
} else LPMS_WARN("No video stream found in input");

if (!ctx->da && (ctx->ai >= 0)) {
ret = open_audio_decoder(ctx, audio_codec);
if (ret < 0) LPMS_ERR(open_input_err, "Unable to open audio decoder")
ctx->last_frame_a = av_frame_alloc();
if (!ctx->last_frame_a) LPMS_ERR(open_input_err, "Unable to alloc last_frame_a");
} else LPMS_WARN("No audio stream found in input");
}

Expand All @@ -306,44 +320,19 @@ int open_input(input_params *params, struct input_ctx *ctx)

void free_input(struct input_ctx *ictx, enum FreeInputPolicy policy)
{
if (FORCE_CLOSE_HW_DECODER == policy) {
// This means we are closing everything, so we also want to
// remove demuxer
if (ictx->ic) avformat_close_input(&ictx->ic);
} else {
// Otherwise we may want to retain demuxer in certain cases. Note that
// this is a lot of effort for very little gain, because demuxer is very
// cheap to create and destroy (being software component)
if (ictx->ic) {
// Only mpegts reuse the demuxer for subsequent segments.
// Close the demuxer for everything else.
// TODO might be reusable with fmp4 ; check!
if (!is_mpegts(ictx->ic)) avformat_close_input(&ictx->ic);
else if (ictx->ic->pb) {
// Reset leftovers from demuxer internals to prepare for next segment
avio_flush(ictx->ic->pb);
avformat_flush(ictx->ic);
avio_closep(&ictx->ic->pb);
}
}
}
if (ictx->ic) avformat_close_input(&ictx->ic);
ictx->flushed = 0;
ictx->flushing = 0;
ictx->pkt_diff = 0;
ictx->sentinel_count = 0;
// this is allocated elsewhere on first video packet
if (ictx->first_pkt) av_packet_free(&ictx->first_pkt);
if (ictx->ac) avcodec_free_context(&ictx->ac);
// video decoder is always closed when it is a SW decoder
// otherwise only when forced
int close_vc = ictx->vc &&
((AV_HWDEVICE_TYPE_NONE == ictx->hw_type) || (FORCE_CLOSE_HW_DECODER == policy));
if (close_vc) {
if (ictx->vc->hw_device_ctx) av_buffer_unref(&ictx->vc->hw_device_ctx);
avcodec_free_context(&ictx->vc);
if (ictx->hw_device_ctx) av_buffer_unref(&ictx->hw_device_ctx);
if (ictx->last_frame_v) av_frame_free(&ictx->last_frame_v);
if ((AV_HWDEVICE_TYPE_NONE == ictx->hw_type) || (FORCE_CLOSE_HW_DECODER == policy)) {
close_video_decoder(ictx);
}
if (ictx->last_frame_a) av_frame_free(&ictx->last_frame_a);

// audio decoder is always closed
close_audio_decoder(ictx);
}

35 changes: 21 additions & 14 deletions ffmpeg/encoder.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ static AVStream *add_stream_copy(struct output_ctx *octx, AVStream *ist)
if (!ist) LPMS_ERR(add_copy_err, "Input stream for copy not available");
AVStream *st = avformat_new_stream(octx->oc, NULL);
if (!st) LPMS_ERR(add_copy_err, "Unable to alloc copy stream");
octx->vi = st->index;
// octx->vi = st->index;
if (octx->fps.den) st->avg_frame_rate = octx->fps;
else st->avg_frame_rate = ist->r_frame_rate;
st->time_base = ist->time_base;
Expand Down Expand Up @@ -45,16 +45,15 @@ static int add_video_stream(struct output_ctx *octx, struct input_ctx *ictx)
{
// video stream to muxer
int ret = 0;
AVStream *st = NULL;
if (is_copy(octx->video->name)) {
// create stream as a copy of existing one
if (ictx->vi < 0) LPMS_ERR(add_video_err, "Input video stream does not exist");
st = add_stream_copy(octx, ictx->ic->streams[ictx->vi]);
if (!st) LPMS_ERR(add_video_err, "Error adding video copy stream");
octx->video_stream = add_stream_copy(octx, ictx->ic->streams[ictx->vi]);
if (!octx->video_stream) LPMS_ERR(add_video_err, "Error adding video copy stream");
} else if (octx->vc) {
// create stream from encoder
st = add_stream_for_encoder(octx, octx->vc);
if (!st) LPMS_ERR(add_video_err, "Error adding video encoder stream");
octx->video_stream = add_stream_for_encoder(octx, octx->vc);
if (!octx->video_stream) LPMS_ERR(add_video_err, "Error adding video encoder stream");
// Video has rescale here. Audio is slightly different
// Rescale the gop/clip time to the expected timebase after filtering.
// The FPS filter outputs pts incrementing by 1 at a rate of 1/framerate
Expand All @@ -74,8 +73,14 @@ static int add_video_stream(struct output_ctx *octx, struct input_ctx *ictx)
octx->clip_to_pts = av_rescale_q(octx->clip_to, ms_tb, dest_tb);
}
} else if (is_drop(octx->video->name)) {
octx->video_stream = NULL;
LPMS_ERR(add_video_err, "add_video_stream called for dropped video!");
} else LPMS_ERR(add_video_err, "No video encoder, not a copy; what is this?");
} else {
// this can actually happen if the transcoder configured for video
// gets segment without actual video stream
octx->video_stream = NULL;
LPMS_WARN("No video encoder, not a copy; missing video input perhaps?");
}

octx->last_video_dts = AV_NOPTS_VALUE;
return 0;
Expand All @@ -96,24 +101,26 @@ static int add_audio_stream(struct input_ctx *ictx, struct output_ctx *octx)

// audio stream to muxer
int ret = 0;
AVStream *st = NULL;
if (is_copy(octx->audio->name)) {
// create stream as a copy of existing one
if (ictx->ai < 0) LPMS_ERR(add_audio_err, "Input audio stream does not exist");
st = add_stream_copy(octx, ictx->ic->streams[ictx->ai]);
octx->audio_stream = add_stream_copy(octx, ictx->ic->streams[ictx->ai]);
} else if (octx->ac) {
// create stream from encoder
st = add_stream_for_encoder(octx, octx->ac);
octx->audio_stream = add_stream_for_encoder(octx, octx->ac);
// Video has rescale here
} else if (is_drop(octx->audio->name)) {
// Supposed to exit this function early if there's a drop
octx->audio_stream = NULL;
LPMS_ERR(add_audio_err, "add_audio_stream called for dropped audio!");
} else {
LPMS_ERR(add_audio_err, "No audio encoder; not a copy; what is this?");
// see comment in add_video_stream above
octx->audio_stream = NULL;
LPMS_WARN("No audio encoder; not a copy; missing audio input perhaps?");
return 0;
}

if (!st) LPMS_ERR(add_audio_err, "Error adding video copy stream");
octx->ai = st->index;
if (!octx->audio_stream) LPMS_ERR(add_audio_err, "Error adding audio stream");;

// Audio has rescale here. Video version is slightly different
AVRational ms_tb = {1, 1000};
Expand All @@ -126,7 +133,7 @@ static int add_audio_stream(struct input_ctx *ictx, struct output_ctx *octx)
}

// signal whether to drop preroll audio
if (st->codecpar->initial_padding) octx->drop_ts = AV_NOPTS_VALUE;
if (octx->audio_stream->codecpar->initial_padding) octx->drop_ts = AV_NOPTS_VALUE;

octx->last_audio_dts = AV_NOPTS_VALUE;

Expand Down
Loading

0 comments on commit 9196c7c

Please sign in to comment.