WIP

robintown · toger5 · commit 53adfa4497b3 · 2025-05-14T19:07:36.000+02:00
diff --git a/docs/controls.md b/docs/controls.md
@@ -1,7 +1,17 @@
 # Global JS controls
 
-A few aspects of Element Call's interface can be controlled through a global API on the `window`:
+A few aspects of Element Call's interface can be controlled through a global API on the `window`.
+
+## Picture-in-picture
 
 - `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode.
 - `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call.
 - `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call.
+
+## Audio output devices
+
+These functions must be used in conjunction with the `controlledOutput` URL parameter in order to have any effect.
+
+- `controls.setOutputDevices(devices: { id: string, name: string }[]): void` Sets the list of available audio outputs.
+- `controls.onOutputDeviceSelect: ((id: string) => void) | undefined` Callback called whenever the user or application selects a new audio output.
+- `controls.setOutputEnabled(enabled: boolean)` Enables/disables all audio output from the application. This can be useful for temporarily pausing audio while the controlling application is switching output devices. Output is enabled by default.
diff --git a/docs/url-params.md b/docs/url-params.md
@@ -63,6 +63,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st
 | `lang`                                                  | [BCP 47](https://www.rfc-editor.org/info/bcp47) code                                                 | No                                                                                                | No                           | The language the app should use.                                                                                                                                                                                                                                                                                                                                                                    |
 | `password`                                              |                                                                                                      | No                                                                                                | No                           | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.)                                                                                                                                                                                                                                                                                       |
 | `perParticipantE2EE`                                    | `true` or `false`                                                                                    | No, defaults to `false`                                                                           | No, defaults to `false`      | Enables per participant encryption with Keys exchanged over encrypted matrix room messages.                                                                                                                                                                                                                                                                                                         |
+| `controlledOutput`                                      | `true` or `false`                                                                                    | No, defaults to `false`                                                                           | No, defaults to `false`      | Whether the [global JS controls for audio output devices](./controls.md#audio-devices) should be enabled, allowing the list of output devices to be controlled by the app hosting Element Call.                                                                                                                                                                                                     |
 | `roomId`                                                | [Matrix Room ID](https://spec.matrix.org/v1.12/appendices/#room-ids)                                 | Yes                                                                                               | No                           | Anything about what room we're pointed to should be from useRoomIdentifier which parses the path and resolves alias with respect to the default server name, however roomId is an exception as we need the room ID in embedded widget mode, and not the room alias (or even the via params because we are not trying to join it). This is also not validated, where it is in `useRoomIdentifier()`. |
 | `showControls`                                          | `true` or `false`                                                                                    | No, defaults to `true`                                                                            | No, defaults to `true`       | Displays controls like mute, screen-share, invite, and hangup buttons during a call.                                                                                                                                                                                                                                                                                                                |
 | `skipLobby` (deprecated: use `intent` instead)          | `true` or `false`                                                                                    | No. If `intent` is explicitly `start_call` then defaults to `true`. Otherwise defaults to `false` | No, defaults to `false`      | Skips the lobby to join a call directly, can be combined with preload in widget. When `true` the audio and video inputs will be muted by default. (This means there currently is no way to start without muted video if one wants to skip the lobby. Also not in widget mode.)                                                                                                                      |
diff --git a/src/App.tsx b/src/App.tsx
@@ -19,10 +19,14 @@ import { ClientProvider } from "./ClientContext";
 import { ErrorPage, LoadingPage } from "./FullScreenView";
 import { DisconnectedBanner } from "./DisconnectedBanner";
 import { Initializer } from "./initializer";
-import { MediaDevicesProvider } from "./livekit/MediaDevicesContext";
+import {
+  ControlledOutputMediaDevicesProvider,
+  MediaDevicesProvider,
+} from "./livekit/MediaDevicesContext";
 import { widget } from "./widget";
 import { useTheme } from "./useTheme";
 import { ProcessorProvider } from "./livekit/TrackProcessorContext";
+import { useUrlParams } from "./UrlParams";
 
 const SentryRoute = Sentry.withSentryReactRouterV7Routing(Route);
 
@@ -51,6 +55,7 @@ const ThemeProvider: FC<SimpleProviderProps> = ({ children }) => {
 };
 
 export const App: FC = () => {
+  const { controlledOutput } = useUrlParams();
   const [loaded, setLoaded] = useState(false);
   useEffect(() => {
     Initializer.init()
@@ -62,6 +67,20 @@ export const App: FC = () => {
       .catch(logger.error);
   });
 
+  const inner = (
+    <Sentry.ErrorBoundary
+      fallback={(error) => <ErrorPage error={error} widget={widget} />}
+    >
+      <DisconnectedBanner />
+      <Routes>
+        <SentryRoute path="/" element={<HomePage />} />
+        <SentryRoute path="/login" element={<LoginPage />} />
+        <SentryRoute path="/register" element={<RegisterPage />} />
+        <SentryRoute path="*" element={<RoomPage />} />
+      </Routes>
+    </Sentry.ErrorBoundary>
+  );
+
   return (
     // eslint-disable-next-line @typescript-eslint/ban-ts-comment
     // @ts-ignore
@@ -72,26 +91,15 @@ export const App: FC = () => {
             {loaded ? (
               <Suspense fallback={null}>
                 <ClientProvider>
-                  <MediaDevicesProvider>
-                    <ProcessorProvider>
-                      <Sentry.ErrorBoundary
-                        fallback={(error) => (
-                          <ErrorPage error={error} widget={widget} />
-                        )}
-                      >
-                        <DisconnectedBanner />
-                        <Routes>
-                          <SentryRoute path="/" element={<HomePage />} />
-                          <SentryRoute path="/login" element={<LoginPage />} />
-                          <SentryRoute
-                            path="/register"
-                            element={<RegisterPage />}
-                          />
-                          <SentryRoute path="*" element={<RoomPage />} />
-                        </Routes>
-                      </Sentry.ErrorBoundary>
-                    </ProcessorProvider>
-                  </MediaDevicesProvider>
+                  <ProcessorProvider>
+                    {controlledOutput ? (
+                      <ControlledOutputMediaDevicesProvider>
+                        {inner}
+                      </ControlledOutputMediaDevicesProvider>
+                    ) : (
+                      <MediaDevicesProvider>{inner}</MediaDevicesProvider>
+                    )}
+                  </ProcessorProvider>
                 </ClientProvider>
               </Suspense>
             ) : (
diff --git a/src/UrlParams.ts b/src/UrlParams.ts
@@ -124,9 +124,15 @@ export interface UrlParams {
    */
   password: string | null;
   /**
-   * Whether we the app should use per participant keys for E2EE.
+   * Whether the app should use per participant keys for E2EE.
    */
   perParticipantE2EE: boolean;
+  /**
+   * Whether the global JS controls for audio output devices should be enabled,
+   * allowing the list of output devices to be controlled by the app hosting
+   * Element Call.
+   */
+  controlledOutput: boolean;
   /**
    * Setting this flag skips the lobby and brings you in the call directly.
    * In the widget this can be combined with preload to pass the device settings
@@ -156,19 +162,16 @@ export interface UrlParams {
    * creating a spa link.
    */
   homeserver: string | null;
-
   /**
    * The user's intent with respect to the call.
    * e.g. if they clicked a Start Call button, this would be `start_call`.
    * If it was a Join Call button, it would be `join_existing`.
    */
   intent: string | null;
-
   /**
    * The rageshake submit URL. This is only used in the embedded package of Element Call.
    */
   rageshakeSubmitUrl: string | null;
-
   /**
    * The Sentry DSN. This is only used in the embedded package of Element Call.
    */
@@ -281,6 +284,7 @@ export const getUrlParams = (
     fontScale: Number.isNaN(fontScale) ? null : fontScale,
     allowIceFallback: parser.getFlagParam("allowIceFallback"),
     perParticipantE2EE: parser.getFlagParam("perParticipantE2EE"),
+    controlledOutput: parser.getFlagParam("controlledMediaDevices"),
     skipLobby: parser.getFlagParam(
       "skipLobby",
       isWidget && intent === UserIntent.StartNewCall,
diff --git a/src/controls.ts b/src/controls.ts
@@ -8,12 +8,22 @@ Please see LICENSE in the repository root for full details.
 import { Subject } from "rxjs";
 
 export interface Controls {
-  canEnterPip: () => boolean;
-  enablePip: () => void;
-  disablePip: () => void;
+  canEnterPip(): boolean;
+  enablePip(): void;
+  disablePip(): void;
+  setOutputDevices(devices: OutputDevice[]): void;
+  onOutputDeviceSelect?: (id: string) => void;
+  setOutputEnabled(enabled: boolean): void;
+}
+
+export interface OutputDevice {
+  id: string;
+  name: string;
 }
 
 export const setPipEnabled$ = new Subject<boolean>();
+export const setOutputDevices = new Subject<OutputDevice[]>();
+export const setOutputEnabled = new Subject<boolean>();
 
 window.controls = {
   canEnterPip(): boolean {
@@ -27,4 +37,14 @@ window.controls = {
     if (!setPipEnabled$.observed) throw new Error("No call is running");
     setPipEnabled$.next(false);
   },
+  setOutputDevices(devices: OutputDevice[]): void {
+    if (!setOutputDevices.observed)
+      throw new Error("Output controls are disabled");
+    setOutputDevices.next(devices);
+  },
+  setOutputEnabled(enabled: boolean): void {
+    if (!setOutputEnabled.observed)
+      throw new Error("Output controls are disabled");
+    setOutputEnabled.next(enabled);
+  },
 };
diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx
@@ -1,5 +1,5 @@
 /*
-Copyright 2023, 2024 New Vector Ltd.
+Copyright 2023-2025 New Vector Ltd.
 
 SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
@@ -18,7 +18,7 @@ import {
 } from "react";
 import { createMediaDeviceObserver } from "@livekit/components-core";
 import { map, startWith } from "rxjs";
-import { useObservableEagerState } from "observable-hooks";
+import { useObservable, useObservableEagerState } from "observable-hooks";
 import { logger } from "matrix-js-sdk/lib/logger";
 
 import {
@@ -29,6 +29,7 @@ import {
   alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
   type Setting,
 } from "../settings/settings";
+import { type OutputDevice, setOutputDevices } from "../controls";
 
 export const EARPIECE_CONFIG_ID = "earpiece-id";
 
@@ -59,12 +60,16 @@ export interface MediaDevice {
   select: (deviceId: string) => void;
 }
 
-export interface MediaDevices {
+interface InputDevices {
   audioInput: MediaDevice;
-  audioOutput: MediaDevice;
   videoInput: MediaDevice;
   startUsingDeviceNames: () => void;
   stopUsingDeviceNames: () => void;
+  usingNames: boolean;
+}
+
+export interface MediaDevices extends Omit<InputDevices, "usingNames"> {
+  audioOutput: MediaDevice;
 }
 
 function useMediaDevice(
@@ -215,11 +220,7 @@ export const devicesStub: MediaDevices = {
 
 export const MediaDevicesContext = createContext<MediaDevices>(devicesStub);
 
-interface Props {
-  children: JSX.Element;
-}
-
-export const MediaDevicesProvider: FC<Props> = ({ children }) => {
+function useInputDevices(): InputDevices {
   // Counts the number of callers currently using device names.
   const [numCallersUsingNames, setNumCallersUsingNames] = useState(0);
   const usingNames = numCallersUsingNames > 0;
@@ -229,11 +230,6 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
     audioInputSetting,
     usingNames,
   );
-  const audioOutput = useMediaDevice(
-    "audiooutput",
-    audioOutputSetting,
-    usingNames,
-  );
   const videoInput = useMediaDevice(
     "videoinput",
     videoInputSetting,
@@ -249,6 +245,115 @@ export const MediaDevicesProvider: FC<Props> = ({ children }) => {
     [setNumCallersUsingNames],
   );
 
+  return {
+    audioInput,
+    videoInput,
+    startUsingDeviceNames,
+    stopUsingDeviceNames,
+    usingNames,
+  };
+}
+
+interface Props {
+  children: JSX.Element;
+}
+
+export const MediaDevicesProvider: FC<Props> = ({ children }) => {
+  const {
+    audioInput,
+    videoInput,
+    startUsingDeviceNames,
+    stopUsingDeviceNames,
+    usingNames,
+  } = useInputDevices();
+
+  const audioOutput = useMediaDevice(
+    "audiooutput",
+    audioOutputSetting,
+    usingNames,
+  );
+
+  const context: MediaDevices = useMemo(
+    () => ({
+      audioInput,
+      audioOutput,
+      videoInput,
+      startUsingDeviceNames,
+      stopUsingDeviceNames,
+    }),
+    [
+      audioInput,
+      audioOutput,
+      videoInput,
+      startUsingDeviceNames,
+      stopUsingDeviceNames,
+    ],
+  );
+
+  return (
+    <MediaDevicesContext.Provider value={context}>
+      {children}
+    </MediaDevicesContext.Provider>
+  );
+};
+
+function useControlledOutput(): MediaDevice {
+  const available = useObservableEagerState(
+    useObservable(() =>
+      setOutputDevices.pipe(
+        startWith<OutputDevice[]>([]),
+        map(
+          (devices) =>
+            new Map<string, DeviceLabel>(
+              devices.map(({ id, name }) => [id, { type: "name", name }]),
+            ),
+        ),
+      ),
+    ),
+  );
+  const [preferredId, select] = useSetting(audioOutputSetting);
+  const selectedId = useMemo(() => {
+    if (available.size) {
+      // If the preferred device is available, use it. Or if every available
+      // device ID is falsy, the browser is probably just being paranoid about
+      // fingerprinting and we should still try using the preferred device.
+      // Worst case it is not available and the browser will gracefully fall
+      // back to some other device for us when requesting the media stream.
+      // Otherwise, select the first available device.
+      return (preferredId !== undefined && available.has(preferredId)) ||
+        (available.size === 1 && available.has(""))
+        ? preferredId
+        : available.keys().next().value;
+    }
+    return undefined;
+  }, [available, preferredId]);
+  useEffect(() => {
+    if (selectedId !== undefined)
+      window.controls.onOutputDeviceSelect?.(selectedId);
+  }, [selectedId]);
+
+  return useMemo(
+    () => ({
+      available,
+      selectedId,
+      selectedGroupId: undefined,
+      select,
+    }),
+    [available, selectedId, select],
+  );
+}
+
+export const ControlledOutputMediaDevicesProvider: FC<Props> = ({
+  children,
+}) => {
+  const {
+    audioInput,
+    videoInput,
+    startUsingDeviceNames,
+    stopUsingDeviceNames,
+  } = useInputDevices();
+  const audioOutput = useControlledOutput();
+
   const context: MediaDevices = useMemo(
     () => ({
       audioInput,
