Reconnect on network change

Author: kamil-stasiak

assets/package-lock.json

@@ -11,7 +11,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@fishjam-dev/react-client": "github:fishjam-dev/react-client-sdk#turnconfig",
+    "@fishjam-dev/react-client": "github:fishjam-dev/react-client-sdk#reconnet-on-network-change",
     "@mediapipe/tasks-vision": "^0.10.12",
     "axios": "^1.6.8",
     "chartist": "^1.3.0",

assets/package.json

@@ -4,15 +4,22 @@ import {
   SCREENSHARING_TRACK_CONSTRAINTS,
   VIDEO_TRACK_CONSTRAINTS
 } from "../../pages/room/consts";
-import { PeerMetadata, TrackMetadata, useCamera, useClient, useMicrophone, useSetupMedia } from "../../fishjam";
-import { ClientEvents, UseCameraResult, SimulcastConfig } from "@fishjam-dev/react-client";
+import {
+  PeerMetadata,
+  TrackMetadata,
+  useCamera,
+  useClient,
+  useMicrophone,
+  useSetupMedia
+} from "../../fishjam";
+import { ClientEvents, CameraAPI, SimulcastConfig } from "@fishjam-dev/react-client";
 import { BlurProcessor } from "./BlurProcessor";
 import { selectBandwidthLimit } from "../../pages/room/bandwidth.tsx";
 import { useDeveloperInfo } from "../../contexts/DeveloperInfoContext.tsx";
 import { useUser } from "../../contexts/UserContext.tsx";
 
 export type LocalPeerContext = {
-  video: UseCameraResult<TrackMetadata>;
+  video: CameraAPI<TrackMetadata>;
   init: () => void;
   blur: boolean;
   setBlur: (status: boolean, restart: boolean) => void;
@@ -67,6 +74,7 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
   });
 
   const video = useCamera();
+  const microphone = useMicrophone();
 
   const [blur, setBlur] = useState(false);
   const processor = useRef<BlurProcessor | null>(null);
@@ -127,6 +135,9 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
         .some((track) => track?.metadata?.type === "camera");
 
       const newTrack = trackRef.current;
+
+      const newMetadata = { active: metadataActive, type: "camera", displayName } as const;
+
       if (!lastCameraTrack && streamRef.current && newTrack) {
         const mediaStream = new MediaStream();
         mediaStream.addTrack(newTrack);
@@ -140,8 +151,7 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
 
         remoteTrackIdRef.current = await client.addTrack(
           newTrack,
-          mediaStream,
-          { active: metadataActive, type: "camera", displayName },
+          newMetadata,
           simulcastConfig,
           selectBandwidthLimit("camera", simulcastEnabled)
         );
@@ -152,7 +162,6 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
         // todo
         //  When you replaceTrack, this does not affect the stream, so the local peer doesn't know that something has changed.
         //  add localTrackReplaced event
-        const newMetadata: TrackMetadata = { active: metadataActive, type: "camera", displayName };
         await client.replaceTrack(remoteTrackIdRef.current, trackRef.current, newMetadata);
       } else if (remoteTrackIdRef.current && !stream) {
         await client.removeTrack(remoteTrackIdRef.current);
@@ -240,12 +249,14 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
       }
     };
 
-    const disconnected: ClientEvents<PeerMetadata, TrackMetadata>["disconnected"] = async (client) => {
+    const disconnected: ClientEvents<PeerMetadata, TrackMetadata>["disconnected"] = async (clientApi) => {
       remoteTrackIdRef.current = null;
 
-      if (client.devices.microphone.stream) client.devices.microphone.stop();
-      if (client.devices.camera.stream) client.devices.camera.stop();
-      if (client.devices.screenShare.stream) client.devices.screenShare.stop();
+      if (!client.isReconnecting()) {
+        if (clientApi.devices.microphone.stream) clientApi.devices.microphone.stop();
+        if (clientApi.devices.camera.stream) clientApi.devices.camera.stop();
+        if (clientApi.devices.screenShare.stream) clientApi.devices.screenShare.stop();
+      }
     };
 
 
@@ -276,12 +287,15 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
 
   const noop: () => Promise<any> = useCallback((..._args) => Promise.resolve(), []);
 
-  const newVideo: UseCameraResult<TrackMetadata> = useMemo(() => ({
+  const newVideo: CameraAPI<TrackMetadata> = useMemo(() => ({
     stream: stream || null,
     track: track || null,
     addTrack: noop,
     removeTrack: noop,
     replaceTrack: noop,
+    updateTrackMetadata: noop,
+    muteTrack: noop,
+    unmuteTrack: noop,
     broadcast: video.broadcast,
     deviceInfo: video.deviceInfo,
     devices: video.devices,
@@ -295,8 +309,6 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
     mediaStatus: video.mediaStatus
   }), [stream, track]);
 
-  const microphone = useMicrophone();
-
   const setDevice = useCallback(async (cameraId: string | null, microphoneId: string | null, blur: boolean) => {
     if (microphoneId && microphoneIntentionRef.current) {
       microphone.start(microphoneId);

assets/src/features/devices/LocalPeerMediaContext.tsx

@@ -1,12 +1,18 @@
 import { createContext, ReactNode, useCallback, useMemo, useState } from "react";
 import Toast from "../components/Toast";
 
-const DEFAULT_TOAST_TIMEOUT = 2500;
+const DEFAULT_TOAST_TIMEOUT = 2500; // milliseconds
 
-export type ToastType = { id: string; message?: string; timeout?: number | "INFINITY"; type?: "information" | "error" };
+export type ToastType = {
+  id: string;
+  message?: string;
+  timeout?: number | "INFINITY"; // milliseconds
+  type?: "information" | "error"
+};
 
 export const ToastContext = createContext({
-  addToast: (newToast: ToastType) => console.error(`Unknown error while adding toast: ${newToast}`)
+  addToast: (newToast: ToastType) => console.error(`Unknown error while adding toast: ${newToast}`),
+  removeToast: (toastId: string) => console.error(`Unknown error while removing toast: ${toastId}`)
 });
 
 export const ToastProvider = ({ children }: { children?: ReactNode }) => {
@@ -28,14 +34,14 @@ export const ToastProvider = ({ children }: { children?: ReactNode }) => {
     [toasts]
   );
 
-  const removeToast = (toastId: string) => {
+  const removeToast = useCallback((toastId: string) => {
     document.getElementById(toastId)?.classList.add("fadeOut");
     setTimeout(() => {
       setToasts((prev) => prev.filter((t) => t.id != toastId));
     }, 2000);
-  };
+  }, []);
 
-  const value = useMemo(() => ({ addToast }), [addToast]);
+  const value = useMemo(() => ({ addToast, removeToast }), [removeToast, addToast]);
 
   return (
     <ToastContext.Provider value={value}>

assets/src/features/shared/context/ToastContext.tsx

@@ -1,11 +1,14 @@
 import { FC, PropsWithChildren, useCallback, useEffect, useState } from "react";
 import useToast from "../shared/hooks/useToast";
 import { ErrorMessage, messageComparator } from "../../pages/room/errorMessage";
-import { useClient } from "../../fishjam";
+import { PeerMetadata, TrackMetadata, useClient } from "../../fishjam";
 import useEffectOnChange from "../shared/hooks/useEffectOnChange";
+import { ClientEvents } from "@fishjam-dev/react-client";
+
+const RECONNECTING_TOAST_ID = "RECONNECTING";
 
 export const StreamingErrorBoundary: FC<PropsWithChildren> = ({ children }) => {
-  const { addToast } = useToast();
+  const { addToast, removeToast } = useToast();
 
   // todo remove state, refactor to function invocation
   const [errorMessage, setErrorMessage] = useState<ErrorMessage | undefined>();
@@ -20,34 +23,104 @@ export const StreamingErrorBoundary: FC<PropsWithChildren> = ({ children }) => {
     [setErrorMessage]
   );
 
+  const displayInfo = useCallback(
+    (text: string, id?: string) => {
+      addToast({
+        id: id || crypto.randomUUID(),
+        message: text,
+        timeout: 5000,
+        type: "information"
+      });
+    },
+    []
+  );
+
   useEffect(() => {
     if (!client) return;
 
-    const onSocketError = (_: Event) => {
+    const onReconnectionStarted: ClientEvents<PeerMetadata, TrackMetadata>["reconnectionStarted"] = () => {
+      console.log("%c" + "reconnectionStarted", "color:green");
+
+      addToast({
+        id: RECONNECTING_TOAST_ID,
+        message: "Reconnecting",
+        timeout: "INFINITY",
+        type: "information"
+      });
+    };
+
+    const onReconnected: ClientEvents<PeerMetadata, TrackMetadata>["reconnected"] = () => {
+      console.log("%cReconnected", "color:green");
+      removeToast(RECONNECTING_TOAST_ID);
+
+      setTimeout(() => {
+        displayInfo("Reconnected");
+      }, 2000);
+    };
+
+    const onReconnectionFailed: ClientEvents<PeerMetadata, TrackMetadata>["reconnectionFailed"] = () => {
+      console.log("%cReconnectionFailed", "color:red");
+      removeToast(RECONNECTING_TOAST_ID);
+
+      setTimeout(() => {
+        handleError(`Reconnection failed`);
+      }, 2000);
+    };
+
+    const onSocketError: ClientEvents<PeerMetadata, TrackMetadata>["socketError"] = (error: Event) => {
+      console.warn(error);
       handleError(`Socket error occurred.`, "onSocketError");
     };
 
-    const onConnectionError = (message: string) => {
-      handleError(`Connection error occurred. ${message ?? ""}`);
+    const onConnectionError: ClientEvents<PeerMetadata, TrackMetadata>["connectionError"] = (error, client) => {
+      if (client.isReconnecting()) {
+        console.log("%c" + "During reconnection: connectionError %o", "color:gray", {
+          error,
+          iceConnectionState: error?.event?.target?.["iceConnectionState"]
+        });
+      } else {
+        console.warn({ error, state: error?.event?.target?.["iceConnectionState"] });
+        handleError(`Connection error occurred. ${error?.message ?? ""}`);
+      }
     };
-    const onJoinError = (_: unknown) => {
+
+    const onJoinError: ClientEvents<PeerMetadata, TrackMetadata>["joinError"] = (event) => {
+      console.log(event);
       handleError(`Failed to join the room`);
     };
-    const onAuthError = () => {
-      handleError(`Socket error occurred.`, "onAuthError");
+
+    const onAuthError: ClientEvents<PeerMetadata, TrackMetadata>["authError"] = (reason) => {
+      if (client.isReconnecting()) {
+        console.log("%c" + "During reconnection: authError: " + reason, "color:gray");
+      } else {
+        handleError(`Socket error occurred.`, "onAuthError");
+      }
     };
 
-    const onSocketClose = (_: Event) => {
-      handleError(`Signaling socket closed.`, "onSocketClose");
+    const onSocketClose: ClientEvents<PeerMetadata, TrackMetadata>["socketClose"] = (event) => {
+      if (client.isReconnecting()) {
+        console.log("%c" + "During reconnection: Signaling socket closed", "color:gray");
+      } else {
+        console.warn(event);
+        handleError(`Signaling socket closed.`, "onSocketClose");
+      }
     };
 
+    client.on("reconnectionStarted", onReconnectionStarted);
+    client.on("reconnected", onReconnected);
+    client.on("reconnectionFailed", onReconnectionFailed);
+
     client.on("socketError", onSocketError);
     client.on("connectionError", onConnectionError);
     client.on("joinError", onJoinError);
     client.on("authError", onAuthError);
     client.on("socketClose", onSocketClose);
 
     return () => {
+      client.off("reconnectionStarted", onReconnectionStarted);
+      client.off("reconnected", onReconnected);
+      client.off("reconnectionFailed", onReconnectionFailed);
+
       client.off("socketError", onSocketError);
       client.off("connectionError", onConnectionError);
       client.off("joinError", onJoinError);

assets/src/features/streaming/StreamingErrorBoundary.tsx

@@ -38,9 +38,9 @@ export const {
   peerMetadataParser: (obj) => peerMetadataSchema.parse(obj),
   trackMetadataParser: (obj) => trackMetadataSchema.parse(obj),
   reconnect: {
-    initialDelay: 10, // ms
-    delay: 200, // ms
-    maxAttempts: 1,
+    initialDelay: 1000, // ms
+    delay: 1000, // ms
+    maxAttempts: 1000,
   }
 }, { storage: true });
 

assets/src/fishjam.ts

@@ -10,6 +10,7 @@ import UnpinnedTilesSection from "./components/StreamPlayer/UnpinnedTilesSection
 import PinnedTilesSection from "./components/StreamPlayer/PinnedTilesSection";
 import useTilePinning from "./hooks/useTilePinning";
 import { toLocalTrackSelector, toRemotePeerSelector, TrackType, useSelector } from "../../fishjam";
+import { useLocalPeer } from "../../features/devices/LocalPeerMediaContext.tsx";
 
 type Props = {
   showSimulcast: boolean;
@@ -36,7 +37,7 @@ const mapRemotePeersToMediaPlayerConfig = (peers: RemotePeer[]): PeerTileConfig[
     const audioTrack: TrackWithId | null = getTrack(peer.tracks, "audio");
 
     return {
-      mediaPlayerId: videoTrack?.remoteTrackId || peer.id,
+      mediaPlayerId: peer.id,
       typeName: "remote",
       peerId: peer.id,
       displayName: peer.displayName || "Unknown",
@@ -108,17 +109,30 @@ export const VideochatSection: FC<Props> = ({ showSimulcast, unpinnedTilesHorizo
     initials: computeInitials(state?.local?.metadata?.name || "")
   }));
 
-  const localUser: PeerTileConfig = {
+  const localPeer = useLocalPeer();
+
+  const localVideoWithId: TrackWithId = useMemo(() =>
+    ({
+      stream: localPeer?.video?.stream ?? undefined,
+      track: localPeer?.video?.track ?? undefined,
+      isSpeaking: false,
+      enabled: localPeer.video.enabled,
+      encodingQuality: null,
+      remoteTrackId: null,
+      metadata: null
+    }), [localPeer]);
+
+  const localUser: PeerTileConfig = useMemo(() => ({
     typeName: "local",
     peerId,
     displayName: LOCAL_PEER_NAME,
     initials,
-    video: video,
+    video: video ?? localVideoWithId,
     audio: audio,
     streamSource: "local",
     mediaPlayerId: LOCAL_VIDEO_ID,
     isSpeaking: false
-  };
+  }), [peerId, initials, video, localVideoWithId, audio]);
 
   const peers: RemotePeer[] = useSelector((state) => toRemotePeerSelector(state));
   const screenSharingStreams = prepareScreenSharingStreams(peers, screenSharing);