Merge pull request #14 from jellyfish-dev/add-screenshare-support

Add screenshare support

lib/compositor_utils.ex

@@ -8,6 +8,7 @@ defmodule RecordingConverter.Compositor do
   @letter_width 12
   @output_width 1280
   @output_height 720
+  @screenshare_ratio 0.8
   @video_output_id "video_output_1"
   @audio_output_id "audio_output_1"
 
@@ -24,7 +25,7 @@ defmodule RecordingConverter.Compositor do
     end
   end
 
-  @spec scene(any()) :: map()
+  @spec scene(list()) :: map()
   def scene(children) do
     %{
       id: "tiles_0",
@@ -43,9 +44,9 @@ defmodule RecordingConverter.Compositor do
   def video_output_id(), do: @video_output_id
 
   @spec generate_output_update(map(), number(), map()) :: [struct()]
-  def generate_output_update(tracks, timestamp, video_tracks_offset),
+  def generate_output_update(tracks, timestamp, camera_tracks_offset),
     do: [
-      generate_video_output_update(tracks, timestamp, video_tracks_offset),
+      generate_video_output_update(tracks, timestamp, camera_tracks_offset),
       generate_audio_output_update(tracks, timestamp)
     ]
 
@@ -82,24 +83,35 @@ defmodule RecordingConverter.Compositor do
   defp generate_video_output_update(
          %{"video" => video_tracks, "audio" => audio_tracks},
          timestamp,
-         video_tracks_offset
+         camera_tracks_offset
        )
        when is_list(video_tracks) do
-    video_tracks_origin = Enum.map(video_tracks, fn track -> track["origin"] end)
+    {camera_tracks, screenshare_tracks} =
+      Enum.split_with(video_tracks, &(get_in(&1, ["metadata", "type"]) != "screensharing"))
+
+    camera_tracks_origin = Enum.map(camera_tracks, fn track -> track["origin"] end)
 
     avatar_tracks =
       Enum.filter(
         audio_tracks,
-        &should_have_avatar?(&1, timestamp, video_tracks_origin, video_tracks_offset)
+        &should_have_avatar?(&1, timestamp, camera_tracks_origin, camera_tracks_offset)
       )
 
-    avatars_config = Enum.map(avatar_tracks, &avatar_view/1)
-    video_tracks_config = Enum.map(video_tracks, &video_input_source_view/1)
+    camera_tracks_config =
+      Enum.map(camera_tracks, &video_input_source_view/1) ++
+        Enum.map(avatar_tracks, &avatar_view/1)
+
+    screenshare_tracks_config = Enum.map(screenshare_tracks, &video_input_source_view/1)
+
+    scene =
+      if screenshare_tracks_config != [],
+        do: scene_with_screenshare(camera_tracks_config, screenshare_tracks_config),
+        else: scene(camera_tracks_config)
 
     %Request.UpdateVideoOutput{
       output_id: @video_output_id,
       schedule_time: Membrane.Time.nanoseconds(timestamp),
-      root: scene(video_tracks_config ++ avatars_config)
+      root: scene
     }
   end
 
@@ -112,26 +124,43 @@ defmodule RecordingConverter.Compositor do
     }
   end
 
+  defp scene_with_screenshare(camera_children, screenshare_children) do
+    %{
+      type: :view,
+      width: @output_width,
+      height: @output_height,
+      direction: :row,
+      children: [
+        %{
+          type: :tiles,
+          width: @output_width * @screenshare_ratio,
+          children: screenshare_children
+        },
+        %{type: :tiles, children: camera_children}
+      ]
+    }
+  end
+
   defp should_have_avatar?(
          %{"origin" => origin} = track,
          timestamp,
-         video_tracks_origin,
-         video_tracks_offset
+         camera_tracks_origin,
+         camera_tracks_offset
        ) do
-    origin not in video_tracks_origin and
+    origin not in camera_tracks_origin and
       longer_than_treshold?(track, timestamp) and
-      not has_video_in_threshold?(origin, video_tracks_offset, timestamp)
+      not has_video_in_threshold?(origin, camera_tracks_offset, timestamp)
   end
 
   defp longer_than_treshold?(%{"offset" => offset} = track, timestamp) do
     ReportParser.calculate_track_end(track, offset) - timestamp > @avatar_threshold_ns
   end
 
-  defp has_video_in_threshold?(origin, video_tracks_offset, timestamp) do
+  defp has_video_in_threshold?(origin, camera_tracks_offset, timestamp) do
     threshold = timestamp + @avatar_threshold_ns
 
     next_video_offset =
-      video_tracks_offset
+      camera_tracks_offset
       |> Map.get(origin, [])
       |> Enum.find(threshold, &(&1 > timestamp))
 

lib/report_parser.ex

@@ -19,10 +19,10 @@ defmodule RecordingConverter.ReportParser do
   @spec get_all_track_actions(tracks :: list()) :: list()
   def get_all_track_actions(tracks) do
     tracks_actions = get_track_actions(tracks)
-    video_tracks_offset = get_video_tracks_offset(tracks)
+    camera_tracks_offset = get_camera_tracks_offset(tracks)
 
     update_scene_notifications =
-      create_update_scene_notifications(tracks_actions, video_tracks_offset)
+      create_update_scene_notifications(tracks_actions, camera_tracks_offset)
 
     unregister_output_actions = generate_unregister_output_actions(tracks_actions)
 
@@ -61,10 +61,11 @@ defmodule RecordingConverter.ReportParser do
     |> Jason.decode!()
   end
 
-  defp get_video_tracks_offset(tracks) do
-    # TODO: remove screenshare tracks, after adding necessary info to reports
+  defp get_camera_tracks_offset(tracks) do
     tracks
-    |> Enum.filter(&(&1["type"] == "video"))
+    |> Enum.filter(
+      &(&1["type"] == "video" and get_in(&1, ["metadata", "type"]) != "screensharing")
+    )
     |> Enum.reduce(%{}, fn %{"origin" => origin, "offset" => offset}, acc ->
       Map.update(acc, origin, [offset], &[offset | &1])
     end)
@@ -84,16 +85,16 @@ defmodule RecordingConverter.ReportParser do
     |> Enum.sort_by(fn {_atom, _track, timestamp} -> timestamp end)
   end
 
-  defp create_update_scene_notifications(track_actions, video_tracks_offset) do
+  defp create_update_scene_notifications(track_actions, camera_tracks_offset) do
     track_actions
     |> Enum.map_reduce(%{"audio" => [], "video" => []}, fn
       {:start, %{"type" => type} = track, timestamp}, acc ->
         acc = Map.update!(acc, type, &[track | &1])
-        {Compositor.generate_output_update(acc, timestamp, video_tracks_offset), acc}
+        {Compositor.generate_output_update(acc, timestamp, camera_tracks_offset), acc}
 
       {:end, %{"type" => type} = track, timestamp}, acc ->
         acc = Map.update!(acc, type, fn tracks -> Enum.reject(tracks, &(&1 == track)) end)
-        {Compositor.generate_output_update(acc, timestamp, video_tracks_offset), acc}
+        {Compositor.generate_output_update(acc, timestamp, camera_tracks_offset), acc}
     end)
     |> then(fn {actions, _acc} -> actions end)
     |> List.flatten()