avcodec/mlpdec: don't depend on context channel layout when setting substream masks

If avctx->ch_layout is unset (as it's allowed and even expeced by the
AV_CODEC_CAP_CHANNEL_CONF flag), the code setting substream masks will fail for
stereo and mono layouts unless a downmix channel was requested.
Fix this by deriving the mask with coded values only.

Fixes issue #20764.

Signed-off-by: James Almer <jamrial@gmail.com>
(cherry picked from commit 530ca627a3)

libavcodec/mlpdec.c

@@ -318,8 +318,10 @@ static av_cold int mlp_decode_init(AVCodecContext *avctx)
             av_channel_layout_uninit(&avctx->ch_layout);
             avctx->ch_layout = (AVChannelLayout)AV_CHANNEL_LAYOUT_5POINT1;
         }
-        else
+        else {
           av_log(avctx, AV_LOG_WARNING, "Invalid downmix layout\n");
+            av_channel_layout_uninit(&m->downmix_layout);
+        }
     }
 
     ff_thread_once(&init_static_once, init_static);
@@ -452,26 +454,22 @@ static int read_major_sync(MLPDecodeContext *m, GetBitContext *gb)
         }
         m->substream[1].mask = mh.channel_layout_thd_stream1;
         if (mh.channels_thd_stream1 == 2 &&
-            mh.channels_thd_stream2 == 2 &&
+            mh.channels_thd_stream2 == 2)
-            m->avctx->ch_layout.nb_channels == 2)
             m->substream[0].mask = AV_CH_LAYOUT_STEREO;
         if ((substr = (mh.num_substreams > 1)))
             m->substream[0].mask = AV_CH_LAYOUT_STEREO;
         if (mh.num_substreams == 1 &&
             mh.channels_thd_stream1 == 1 &&
-            mh.channels_thd_stream2 == 1 &&
+            mh.channels_thd_stream2 == 1)
-            m->avctx->ch_layout.nb_channels == 1)
             m->substream[0].mask = AV_CH_LAYOUT_MONO;
         if (mh.num_substreams > 2)
             if (mh.channel_layout_thd_stream2)
                 m->substream[2].mask = mh.channel_layout_thd_stream2;
             else
                 m->substream[2].mask = mh.channel_layout_thd_stream1;
-        if (m->avctx->ch_layout.nb_channels > 2)
+        if (mh.num_substreams == 2 && (!m->downmix_layout.nb_channels ||
-            if (mh.num_substreams > 2)
+                                        m->downmix_layout.nb_channels > 2))
-                m->substream[1].mask = mh.channel_layout_thd_stream1;
+            m->substream[1].mask = mh.channel_layout_thd_stream2;
-            else
-                m->substream[mh.num_substreams > 1].mask = mh.channel_layout_thd_stream2;
     }
 
     m->needs_reordering = mh.channel_arrangement >= 18 && mh.channel_arrangement <= 20;

libavformat/demux.c

@@ -1167,7 +1167,10 @@ static int parse_packet(AVFormatContext *s, AVPacket *pkt,
     AVPacket *out_pkt = si->parse_pkt;
     AVStream *st = s->streams[stream_index];
     FFStream *const sti = ffstream(st);
+    const AVPacketSideData *sd = NULL;
     const uint8_t *data = pkt->data;
+    uint8_t *extradata = sti->avctx->extradata;
+    int extradata_size = sti->avctx->extradata_size;
     int size = pkt->size;
     int ret = 0, got_output = flush;
 
@@ -1184,6 +1187,16 @@ static int parse_packet(AVFormatContext *s, AVPacket *pkt,
         }
     }
 
+    if (pkt->side_data_elems)
+        sd = av_packet_side_data_get(pkt->side_data, pkt->side_data_elems,
+                                     AV_PKT_DATA_NEW_EXTRADATA);
+    if (sd) {
+        av_assert1(size && !flush);
+
+        sti->avctx->extradata      = sd->data;
+        sti->avctx->extradata_size = sd->size;
+    }
+
     while (size > 0 || (flush && got_output)) {
         int64_t next_pts = pkt->pts;
         int64_t next_dts = pkt->dts;
@@ -1277,6 +1290,11 @@ static int parse_packet(AVFormatContext *s, AVPacket *pkt,
     }
 
 fail:
+    if (sd) {
+        sti->avctx->extradata      = extradata;
+        sti->avctx->extradata_size = extradata_size;
+    }
+
     if (ret < 0)
         av_packet_unref(out_pkt);
     av_packet_unref(pkt);
@@ -1370,6 +1388,11 @@ static int read_frame_internal(AVFormatContext *s, AVPacket *pkt)
 
         st->event_flags |= AVSTREAM_EVENT_FLAG_NEW_PACKETS;
 
+        int new_extradata = !!av_packet_side_data_get(pkt->side_data, pkt->side_data_elems,
+                                                      AV_PKT_DATA_NEW_EXTRADATA);
+        if (new_extradata)
+            sti->need_context_update = 1;
+
         /* update context if required */
         if (sti->need_context_update) {
             if (avcodec_is_open(sti->avctx)) {
@@ -1380,8 +1403,9 @@ static int read_frame_internal(AVFormatContext *s, AVPacket *pkt)
                     return ret;
             }
 
-            /* close parser, because it depends on the codec */
+            /* close parser, because it depends on the codec and extradata */
-            if (sti->parser && sti->avctx->codec_id != st->codecpar->codec_id) {
+            if (sti->parser &&
+                (sti->avctx->codec_id != st->codecpar->codec_id || new_extradata)) {
                 av_parser_close(sti->parser);
                 sti->parser = NULL;
             }