Fix subtitle syncing

Author: philipgiuliani

lib/membrane/hls/sink_bin.ex

@@ -195,11 +195,11 @@ defmodule Membrane.HLS.SinkBin do
       |> child({:segments, track_id}, %Membrane.WebVTT.SegmentFilter{
         segment_duration: state.opts.target_segment_duration - Membrane.Time.second(),
         headers: [
-          %Subtitle.WebVTT.HeaderLine{key: :description, original: "WEBVTT"},
-          %Subtitle.WebVTT.HeaderLine{
-            key: :x_timestamp_map,
-            original: "X-TIMESTAMP-MAP=LOCAL:00:00:00.000,MPEGTS:90000"
-          }
+          %Subtitle.WebVTT.HeaderLine{key: :description, original: "WEBVTT"}
+          # %Subtitle.WebVTT.HeaderLine{
+          #   key: :x_timestamp_map,
+          #   original: "X-TIMESTAMP-MAP=LOCAL:00:00:00.000,MPEGTS:90000"
+          # }
         ]
       })
       |> child({:sink, track_id}, %Membrane.HLS.WebVTTSink{

mix.exs

@@ -31,7 +31,8 @@ defmodule Membrane.HLS.MixProject do
       {:kim_q, "~> 1.0"},
       {:kim_hls, github: "kim-company/kim_hls"},
       {:membrane_text_format, github: "kim-company/membrane_text_format"},
-      {:membrane_webvtt_plugin, github: "kim-company/membrane_webvtt_plugin"}
+      {:membrane_webvtt_plugin, github: "kim-company/membrane_webvtt_plugin"},
+      {:membrane_flv_plugin, "~> 0.13", only: :test}
     ]
   end
 

mix.lock

@@ -12,6 +12,7 @@
   "membrane_cmaf_format": {:hex, :membrane_cmaf_format, "0.7.1", "9ea858faefdcb181cdfa8001be827c35c5f854e9809ad57d7062cff1f0f703fd", [:mix], [], "hexpm", "3c7b4ed2a986e27f6f336d2f19e9442cb31d93b3142fc024c019572faca54a73"},
   "membrane_core": {:hex, :membrane_core, "1.1.1", "4dcff6e9f3b2ecd4f437c20e201e53957731772c0f15b3005062c41f7f58f500", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:qex, "~> 0.3", [hex: :qex, repo: "hexpm", optional: false]}, {:ratio, "~> 3.0 or ~> 4.0", [hex: :ratio, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3802f3fc071505c59d48792487d9927e803d4edb4039710ffa52cdb60bb0aecc"},
   "membrane_file_plugin": {:hex, :membrane_file_plugin, "0.17.2", "650e134c2345d946f930082fac8bac9f5aba785a7817d38a9a9da41ffc56fa92", [:mix], [{:logger_backends, "~> 1.0", [hex: :logger_backends, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "df50c6040004cd7b901cf057bd7e99c875bbbd6ae574efc93b2c753c96f43b9d"},
+  "membrane_flv_plugin": {:hex, :membrane_flv_plugin, "0.13.0", "faa0c9e6a6da1f1625f3744cd49fadd49958da56124fc0a1c6feb79fe182d386", [:mix], [{:membrane_aac_format, "~> 0.8.0", [hex: :membrane_aac_format, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.1", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_timestamp_queue, "~> 0.2.1", [hex: :membrane_timestamp_queue, repo: "hexpm", optional: false]}], "hexpm", "cc950913e25ab5caf5057cd5b783307b219807320619e0a3a7ec76522c53cf74"},
   "membrane_h264_format": {:hex, :membrane_h264_format, "0.6.1", "44836cd9de0abe989b146df1e114507787efc0cf0da2368f17a10c47b4e0738c", [:mix], [], "hexpm", "4b79be56465a876d2eac2c3af99e115374bbdc03eb1dea4f696ee9a8033cd4b0"},
   "membrane_h265_format": {:hex, :membrane_h265_format, "0.2.0", "1903c072cf7b0980c4d0c117ab61a2cd33e88782b696290de29570a7fab34819", [:mix], [], "hexpm", "6df418bdf242c0d9f7dbf2e5aea4c2d182e34ac9ad5a8b8cef2610c290002e83"},
   "membrane_h26x_plugin": {:hex, :membrane_h26x_plugin, "0.10.2", "caf2790d8c107df35f8d456b45f4e09fb9c56ce6c7669a3a03f7d59972e6ed82", [:mix], [{:bunch, "~> 1.4", [hex: :bunch, repo: "hexpm", optional: false]}, {:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}, {:membrane_h264_format, "~> 0.6.0", [hex: :membrane_h264_format, repo: "hexpm", optional: false]}, {:membrane_h265_format, "~> 0.2.0", [hex: :membrane_h265_format, repo: "hexpm", optional: false]}], "hexpm", "becf1ac4a589adecd850137ccd61a33058f686083a514a7e39fcd721bcf9fb2e"},

test/fixtures/avsync.flv

@@ -13,10 +13,15 @@ defmodule Membrane.HLS.SinkBinTest do
         storage: HLS.Storage.File.new()
       }),
 
-      # Audio
-      child(:aac_source, %Membrane.File.Source{
-        location: "test/fixtures/samples_big-buck-bunny_bun33s.aac"
+      # Source
+      child(:source, %Membrane.File.Source{
+        location: "test/fixtures/avsync.flv"
       })
+      |> child(:demuxer, Membrane.FLV.Demuxer),
+
+      # Audio
+      get_child(:demuxer)
+      |> via_out(Pad.ref(:audio, 0))
       |> child(:aac_parser, %Membrane.AAC.Parser{
         out_encapsulation: :none,
         output_config: :esds
@@ -50,24 +55,24 @@ defmodule Membrane.HLS.SinkBinTest do
             metadata: %{to: Membrane.Time.milliseconds(99)}
           },
           %Membrane.Buffer{
-            payload: "Subtitle from start to 5s",
+            payload: "Subtitle from start to 6s",
             pts: Membrane.Time.milliseconds(100),
-            metadata: %{to: Membrane.Time.seconds(5)}
+            metadata: %{to: Membrane.Time.seconds(6) - Membrane.Time.millisecond()}
           },
           %Membrane.Buffer{
             payload: "",
-            pts: Membrane.Time.seconds(5) + Membrane.Time.millisecond(),
-            metadata: %{to: Membrane.Time.seconds(11)}
+            pts: Membrane.Time.seconds(6),
+            metadata: %{to: Membrane.Time.seconds(12) - Membrane.Time.millisecond()}
           },
           %Membrane.Buffer{
-            payload: "Subtitle from 11s to 15s",
-            pts: Membrane.Time.seconds(11) + Membrane.Time.millisecond(),
-            metadata: %{to: Membrane.Time.seconds(15)}
+            payload: "Subtitle from 12s to 16s",
+            pts: Membrane.Time.seconds(12),
+            metadata: %{to: Membrane.Time.seconds(16) - Membrane.Time.millisecond()}
           },
           %Membrane.Buffer{
             payload: "",
-            pts: Membrane.Time.seconds(15) + Membrane.Time.millisecond(),
-            metadata: %{to: Membrane.Time.seconds(32)}
+            pts: Membrane.Time.seconds(16),
+            metadata: %{to: Membrane.Time.seconds(30) - Membrane.Time.millisecond()}
           }
         ]
       })
@@ -90,9 +95,9 @@ defmodule Membrane.HLS.SinkBinTest do
       |> get_child(:sink),
 
       # Video
-      child(:h264_source, %Membrane.File.Source{
-        location: "test/fixtures/samples_big-buck-bunny_bun33s_720x480.h264"
-      })
+      # Audio
+      get_child(:demuxer)
+      |> via_out(Pad.ref(:video, 0))
       |> child(:h264_parser, %Membrane.H264.Parser{
         generate_best_effort_timestamps: %{framerate: {25, 1}},
         output_stream_structure: :avc1
@@ -105,7 +110,7 @@ defmodule Membrane.HLS.SinkBinTest do
               uri: uri,
               bandwidth: 850_000,
               resolution: format.resolution,
-              frame_rate: 25.0,
+              frame_rate: 30.0,
               codecs: [],
               audio: "audio",
               subtitles: "subtitles"