Fix pixel format detection for VP8/VP9 streams (#1070)

Author: j-b-m

src/modules/avformat/producer_avformat.c

@@ -490,13 +490,44 @@ static mlt_properties find_default_streams(producer_avformat self)
     // Allow for multiple audio and video streams in the file and select first of each (if available)
     for (i = 0; i < context->nb_streams; i++) {
         // Get the codec context
+        AVFormatContext *vpx_context = NULL;
         AVStream *stream = context->streams[i];
         if (!stream)
             continue;
         AVCodecParameters *codec_params = stream->codecpar;
         const AVCodec *codec = avcodec_find_decoder(codec_params->codec_id);
         if (!codec)
             continue;
+        int switch_to_vpx = 0;
+        if (codec_params->codec_id == AV_CODEC_ID_VP9) {
+            if (!(codec = avcodec_find_decoder_by_name("libvpx-vp9"))) {
+                codec = avcodec_find_decoder(codec_params->codec_id);
+            } else {
+                switch_to_vpx = 1;
+            }
+        } else if (codec_params->codec_id == AV_CODEC_ID_VP8) {
+            if (!(codec = avcodec_find_decoder_by_name("libvpx"))) {
+                codec = avcodec_find_decoder(codec_params->codec_id);
+            } else {
+                switch_to_vpx = 1;
+            }
+        }
+        if (switch_to_vpx) {
+            // Use a temporary format context to get the real pixel format with the libvpx decoder,
+            // since the native decoder incorreclty detects yuva420p as yuv420p
+            int error = avformat_open_input(&vpx_context,
+                                            mlt_properties_get(meta_media, "resource"),
+                                            NULL,
+                                            NULL);
+            if (!error) {
+                vpx_context->video_codec = codec;
+                avformat_find_stream_info(vpx_context, NULL);
+                AVStream *vpx_stream = vpx_context->streams[i];
+                if (vpx_stream) {
+                    codec_params = vpx_stream->codecpar;
+                }
+            }
+        }
 
         snprintf(key, sizeof(key), "meta.media.%u.stream.type", i);
 
@@ -634,6 +665,10 @@ static mlt_properties find_default_streams(producer_avformat self)
                 free(value);
             }
         }
+        if (vpx_context) {
+            avformat_close_input(&vpx_context);
+            vpx_context = NULL;
+        }
     }
 
     // Use the album art if that is all we have