Fix simulcast

Author: kamil-stasiak

assets/src/features/devices/LocalPeerMediaContext.tsx

@@ -10,6 +10,7 @@ import { BlurProcessor } from "./BlurProcessor";
 import { selectBandwidthLimit } from "../../pages/room/bandwidth.tsx";
 import { useDeveloperInfo } from "../../contexts/DeveloperInfoContext.tsx";
 import EmptyVideoWorker from "./emptyVideoWorker.ts?worker";
+import { SimulcastConfig } from "@jellyfish-dev/ts-client-sdk";
 
 export type LocalPeerContext = {
   video: UseCameraResult<TrackMetadata>;
@@ -73,6 +74,10 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
   const { simulcast } = useDeveloperInfo();
   const simulcastEnabled = simulcast.status;
 
+  useEffect(() => {
+    console.log({ simulcastEnabled });
+  }, []);
+
   const blurRef = useRef<boolean>(false);
 
   const [stream, setStream] = useState<MediaStream | null>(null);
@@ -129,20 +134,25 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
 
         broadcastedStreamRef.current = mediaStream;
 
+        const simulcastConfig: SimulcastConfig | undefined =
+          simulcastEnabled
+            ? { enabled: true, activeEncodings: ["l", "m", "h"], disabledEncodings: [] }
+            : undefined;
+
         remoteTrackIdRef.current = await client.addTrack(
           trackRef.current,
           mediaStream,
-          // todo think about track.enabled
           { active: metadataActive, type: "camera" },
-          simulcastEnabled ? { enabled: true, activeEncodings: ["l", "m", "h"], disabledEncodings: [] } : undefined,
+          simulcastConfig,
           selectBandwidthLimit("camera", simulcastEnabled)
         );
       } else if (remoteTrackIdRef.current && trackRef.current) {
-        // todo add setter as an alternative to setting whole object
         broadcastedStreamRef.current?.removeTrack(broadcastedStreamRef.current?.getVideoTracks()[0]);
         broadcastedStreamRef.current?.addTrack(trackRef.current);
 
-        // todo when you replaceTrack this not affects stream so local peer don't know that something changes
+        // todo
+        //  When you replaceTrack, this does not affect the stream, so the local peer doesn't know that something has changed.
+        //  add localTrackReplaced event
         const newMetadata: TrackMetadata = { active: metadataActive, type: "camera" };
 
         await client.replaceTrack(remoteTrackIdRef.current, trackRef.current, newMetadata);
@@ -151,7 +161,7 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
         remoteTrackIdRef.current = null;
       }
     }
-  }, [setStream, setTrack]);
+  }, [setStream, setTrack, simulcastEnabled]);
 
   useEffect(() => {
     const managerInitialized: ClientEvents<PeerMetadata, TrackMetadata>["managerInitialized"] = (event) => {
@@ -163,35 +173,13 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
       trackRef.current = event.video?.media?.track || null;
       streamRef.current = event.video?.media?.stream || null;
 
-      const snapshot = client;
-
-      const cameraId = snapshot?.media?.video?.media?.deviceInfo?.deviceId || null;
+      const cameraId = client?.media?.video?.media?.deviceInfo?.deviceId || null;
 
       if (cameraId) {
         lastCameraIdRef.current = cameraId;
       }
     };
 
-
-    /*
-     * Przypadki:
-     * 1) uruchamiam kamerę i następnie wchodze do pokoju
-     *    - intencja: true, ma być włączona
-     *    - leci event
-     *        - disconnect (czyli wyłączam)
-     *        - i następnie joined (czyli uruchamiam bo intencja true)
-     *    - sprawdzam intencje i wiem, że muszę uruchomić ponownie
-     * 2) Wychodzę z pokoju i wracam
-     *    - leci event disconnect -> czyli wyłączam
-     *    - intencja jest true
-     *    - wracam do pokoju czyli leci
-     *        - connect (uruchamiam bo intencja jest true)
-     *        - disconnect (wyłączam)
-     *        - connect uruchamiam (bo inencja jest true)
-     *
-     * Czy z tego powodu wynika, że za każdym razem jak uruchamiam urządzenie, to sprawdzam czy jestem joined
-     * i jak tak to dodaję
-     */
     const joinedHandler: ClientEvents<PeerMetadata, TrackMetadata>["joined"] = async () => {
       const stream = client.devices.camera.stream;
       const track = client.devices.camera.track;

assets/src/features/room-page/components/Navbar.tsx

@@ -10,7 +10,6 @@ import Settings from "../icons/Settings";
 import MediaControlButton from "../../../pages/room/components/MediaControlButton";
 import { useModal } from "../../../contexts/ModalContext";
 import useSmartphoneViewport from "../../shared/hooks/useSmartphoneViewport";
-import { useStatus } from "../../../jellyfish.types";
 
 const Navbar: FC = () => {
   const match = useParams();
@@ -26,8 +25,6 @@ const Navbar: FC = () => {
 
   const isSmartphone = useSmartphoneViewport().isSmartphone;
 
-  const status = useStatus();
-
   return (
     <div className="flex w-full max-w-full flex-row justify-between gap-y-4">
       {!isSmartphone ? (
@@ -40,7 +37,7 @@ const Navbar: FC = () => {
       <div className={clsx("flex flex-row items-center gap-x-3 font-aktivGrotesk")}>
         {!isSmartphone && (
           <>
-            <span>({status})Invite link</span>
+            <span>Invite link</span>
             <Button
               onClick={onLinkCopy}
               variant="light"

assets/src/features/streaming/StreamingContext.tsx

@@ -36,9 +36,9 @@ export const {
   peerMetadataParser: (obj) => peerMetadataSchema.parse(obj),
   trackMetadataParser: (obj) => trackMetadataSchema.parse(obj),
   reconnect: {
-    initialDelay: 5,
+    initialDelay: 10, // ms
+    delay: 200, // ms
     maxAttempts: 1,
-    delay: 5
   }
 }, { storage: true });
 

assets/src/jellyfish.types.ts

@@ -1,10 +1,10 @@
-// import React from "react";
+import React from "react";
 import ReactDOM from "react-dom/client";
 import App from "./App.tsx";
 import "./index.css";
 
 ReactDOM.createRoot(document.getElementById("root") as HTMLElement).render(
-  // <React.StrictMode>
+  <React.StrictMode>
     <App />
-  // </React.StrictMode>
+  </React.StrictMode>
 );

assets/src/main.tsx

@@ -129,10 +129,6 @@ const ConnectComponent: FC<ConnectComponentProps> = (
 
     client?.on("joined", callback);
 
-    window["ws-close"] = () => {
-      client["client"]["websocket"].close();
-    };
-
     return () => {
       client?.removeListener("joined", callback);
       intervalId && clearInterval(intervalId);
@@ -144,9 +140,6 @@ const ConnectComponent: FC<ConnectComponentProps> = (
   useEffect(() => {
     getTokenAndAddress(roomId)
       .then((tokenAndAddress) => {
-
-        // todo make connect in membrane js idempotent
-        // because i get two connect event from this hook
         return connect({
           peerMetadata: { name: username },
           token: tokenAndAddress.token,

assets/src/pages/room/RoomPage.tsx

@@ -127,6 +127,7 @@ const getAutomaticControls = (
 ];
 
 //dev helpers
+// todo fix manual mode
 const getManualControls = (
   navigate: NavigateFunction,
   microphone: UseMicrophoneResult<TrackMetadata>,

assets/src/pages/room/components/MediaControlButtons.tsx

@@ -23,7 +23,7 @@ export const useSimulcastRemoteEncoding = (
       setTargetEncodingState(encoding);
 
       if (!trackId || !peerId || !client) return;
-      // client.setTargetTrackEncoding(trackId, encoding);
+      client.setTargetTrackEncoding(trackId, encoding);
     },
     [peerId, trackId, client]
   );