Recover from RTC errors

Author: kamil-stasiak

assets/package-lock.json

@@ -11,7 +11,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
-    "@fishjam-dev/react-client": "^0.4.0",
+    "@fishjam-dev/react-client": "github:fishjam-dev/react-client-sdk",
     "@mediapipe/tasks-vision": "^0.10.12",
     "axios": "^1.6.8",
     "chartist": "^1.3.0",

assets/package.json

@@ -5,14 +5,14 @@ import {
   VIDEO_TRACK_CONSTRAINTS
 } from "../../pages/room/consts";
 import { PeerMetadata, TrackMetadata, useCamera, useClient, useMicrophone, useSetupMedia } from "../../fishjam";
-import { ClientEvents, UseCameraResult, SimulcastConfig } from "@fishjam-dev/react-client";
+import { ClientEvents, CameraAPI, SimulcastConfig } from "@fishjam-dev/react-client";
 import { BlurProcessor } from "./BlurProcessor";
 import { selectBandwidthLimit } from "../../pages/room/bandwidth.tsx";
 import { useDeveloperInfo } from "../../contexts/DeveloperInfoContext.tsx";
 import { useUser } from "../../contexts/UserContext.tsx";
 
 export type LocalPeerContext = {
-  video: UseCameraResult<TrackMetadata>;
+  video: CameraAPI<TrackMetadata>;
   init: () => void;
   blur: boolean;
   setBlur: (status: boolean, restart: boolean) => void;
@@ -67,6 +67,7 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
   });
 
   const video = useCamera();
+  const microphone = useMicrophone();
 
   const [blur, setBlur] = useState(false);
   const processor = useRef<BlurProcessor | null>(null);
@@ -127,6 +128,9 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
         .some((track) => track?.metadata?.type === "camera");
 
       const newTrack = trackRef.current;
+
+      const newMetadata = { active: metadataActive, type: "camera", displayName } as const;
+
       if (!lastCameraTrack && streamRef.current && newTrack) {
         const mediaStream = new MediaStream();
         mediaStream.addTrack(newTrack);
@@ -140,8 +144,7 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
 
         remoteTrackIdRef.current = await client.addTrack(
           newTrack,
-          mediaStream,
-          { active: metadataActive, type: "camera", displayName },
+          newMetadata,
           simulcastConfig,
           selectBandwidthLimit("camera", simulcastEnabled)
         );
@@ -152,7 +155,6 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
         // todo
         //  When you replaceTrack, this does not affect the stream, so the local peer doesn't know that something has changed.
         //  add localTrackReplaced event
-        const newMetadata: TrackMetadata = { active: metadataActive, type: "camera", displayName };
         await client.replaceTrack(remoteTrackIdRef.current, trackRef.current, newMetadata);
       } else if (remoteTrackIdRef.current && !stream) {
         await client.removeTrack(remoteTrackIdRef.current);
@@ -263,7 +265,6 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
       client.removeListener("devicesReady", devicesReady);
       client.removeListener("deviceStopped", deviceStopped);
       client.removeListener("disconnected", disconnected);
-
     };
   }, [simulcast.status]);
 
@@ -276,12 +277,15 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
 
   const noop: () => Promise<any> = useCallback((..._args) => Promise.resolve(), []);
 
-  const newVideo: UseCameraResult<TrackMetadata> = useMemo(() => ({
+  const newVideo: CameraAPI<TrackMetadata> = useMemo(() => ({
     stream: stream || null,
     track: track || null,
     addTrack: noop,
     removeTrack: noop,
     replaceTrack: noop,
+    updateTrackMetadata: noop,
+    muteTrack: noop,
+    unmuteTrack: noop,
     broadcast: video.broadcast,
     deviceInfo: video.deviceInfo,
     devices: video.devices,
@@ -295,8 +299,6 @@ export const LocalPeerMediaProvider = ({ children }: Props) => {
     mediaStatus: video.mediaStatus
   }), [stream, track]);
 
-  const microphone = useMicrophone();
-
   const setDevice = useCallback(async (cameraId: string | null, microphoneId: string | null, blur: boolean) => {
     if (microphoneId && microphoneIntentionRef.current) {
       microphone.start(microphoneId);

assets/src/features/devices/LocalPeerMediaContext.tsx

@@ -38,9 +38,9 @@ export const {
   peerMetadataParser: (obj) => peerMetadataSchema.parse(obj),
   trackMetadataParser: (obj) => trackMetadataSchema.parse(obj),
   reconnect: {
-    initialDelay: 10, // ms
-    delay: 200, // ms
-    maxAttempts: 1,
+    initialDelay: 200, // ms
+    delay: 500, // ms
+    maxAttempts: 5,
   }
 }, { storage: true });
 

assets/src/fishjam.ts

@@ -22,7 +22,7 @@ import {
   useMicrophone,
   useScreenShare
 } from "../../../fishjam.ts";
-import { UseCameraResult, UseMicrophoneResult, UseScreenShareResult, Client } from "@fishjam-dev/react-client";
+import { CameraAPI, MicrophoneAPI, ScreenShareAPI, Client } from "@fishjam-dev/react-client";
 import { LocalPeerContext, useLocalPeer } from "../../../features/devices/LocalPeerMediaContext.tsx";
 import { useUser } from "../../../contexts/UserContext.tsx";
 
@@ -40,9 +40,9 @@ const getAutomaticControls = (
   isSidebarOpen: boolean,
   openSidebar: () => void,
   isMobileViewport: boolean,
-  microphone: UseMicrophoneResult<TrackMetadata>,
-  screenShare: UseScreenShareResult<TrackMetadata>,
-  camera: UseCameraResult<TrackMetadata>,
+  microphone: MicrophoneAPI<TrackMetadata>,
+  screenShare: ScreenShareAPI<TrackMetadata>,
+  camera: CameraAPI<TrackMetadata>,
   localPeerContext: LocalPeerContext
 ): ControlButton[] => [
   camera.stream
@@ -130,9 +130,9 @@ const getAutomaticControls = (
 // todo fix manual mode
 const getManualControls = (
   navigate: NavigateFunction,
-  microphone: UseMicrophoneResult<TrackMetadata>,
-  screenShare: UseScreenShareResult<TrackMetadata>,
-  camera: UseCameraResult<TrackMetadata>,
+  microphone: MicrophoneAPI<TrackMetadata>,
+  screenShare: ScreenShareAPI<TrackMetadata>,
+  camera: CameraAPI<TrackMetadata>,
   client: Client<PeerMetadata, TrackMetadata>,
   displayName: string,
   roomId?: string
@@ -402,9 +402,9 @@ const MediaControlButtons: FC<MediaControlButtonsProps> =
 
     const navigate = useNavigate();
 
-    const camera: UseCameraResult<TrackMetadata> = useCamera();
-    const microphone: UseMicrophoneResult<TrackMetadata> = useMicrophone();
-    const screenShare: UseScreenShareResult<TrackMetadata> = useScreenShare();
+    const camera: CameraAPI<TrackMetadata> = useCamera();
+    const microphone: MicrophoneAPI<TrackMetadata> = useMicrophone();
+    const screenShare: ScreenShareAPI<TrackMetadata> = useScreenShare();
     const client = useClient();
 
     const localPeerContext = useLocalPeer();