From a1312fb4ff0071a4300a036375294939256a37ad Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 22 May 2025 19:21:04 +0200 Subject: [PATCH 1/3] backport: Audio output controls for mobile native audio device selection #3270 --- dev-backend-docker-compose.yml | 2 +- docs/controls.md | 16 +- docs/url-params.md | 1 + locales/en/app.json | 1 + package.json | 2 +- src/UrlParams.ts | 14 +- src/controls.ts | 72 +++++++- src/livekit/MediaDevicesContext.tsx | 254 ++++++++++++++++++++-------- src/livekit/useLivekit.ts | 7 +- src/room/GroupCallView.tsx | 7 +- src/room/InCallView.tsx | 4 +- src/room/MuteStates.test.tsx | 4 +- src/room/MuteStates.ts | 4 +- src/settings/DeviceSelection.tsx | 4 +- src/settings/SettingsModal.tsx | 42 ++++- src/settings/settings.ts | 2 +- src/state/MuteAllAudioModel.test.ts | 36 ++++ src/state/MuteAllAudioModel.ts | 19 +++ src/useAudioContext.test.tsx | 2 +- yarn.lock | 14 +- 20 files changed, 399 insertions(+), 108 deletions(-) create mode 100644 src/state/MuteAllAudioModel.test.ts create mode 100644 src/state/MuteAllAudioModel.ts diff --git a/dev-backend-docker-compose.yml b/dev-backend-docker-compose.yml index 07a608637..da3c3530a 100644 --- a/dev-backend-docker-compose.yml +++ b/dev-backend-docker-compose.yml @@ -6,7 +6,7 @@ services: image: ghcr.io/element-hq/lk-jwt-service:latest-ci hostname: auth-server environment: - - LK_JWT_PORT=8080 + - LIVEKIT_JWT_PORT=8080 - LIVEKIT_URL=wss://matrix-rtc.m.localhost/livekit/sfu - LIVEKIT_KEY=devkey - LIVEKIT_SECRET=secret diff --git a/docs/controls.md b/docs/controls.md index 02df61efd..bb4572378 100644 --- a/docs/controls.md +++ b/docs/controls.md @@ -1,7 +1,21 @@ # Global JS controls -A few aspects of Element Call's interface can be controlled through a global API on the `window`: +A few aspects of Element Call's interface can be controlled through a global API on the `window`. + +## Picture-in-picture - `controls.canEnterPip(): boolean` Determines whether it's possible to enter picture-in-picture mode. - `controls.enablePip(): void` Puts the call interface into picture-in-picture mode. Throws if not in a call. - `controls.disablePip(): void` Takes the call interface out of picture-in-picture mode, restoring it to its natural display mode. Throws if not in a call. + +## Audio devices + +On mobile platforms (iOS, Android), web views do not reliably support selecting audio output devices such as the main speaker, earpiece, or headset. To address this limitation, the following functions allow the hosting application (e.g., Element Web, Element X) to manage audio devices via exposed JavaScript interfaces. These functions must be enabled using the URL parameter `controlledAudioDevices` to take effect. + +- `controls.setAvailableAudioDevices(devices: { id: string, name: string, forEarpiece?: boolean, isEarpiece?: boolean isSpeaker?: boolean, isExternalHeadset?, boolean; }[]): void` Sets the list of available audio outputs. `forEarpiece` is used on iOS only. + It flags the device that should be used if the user selects earpiece mode. This should be the main stereo loudspeaker of the device. +- `controls.onAudioDeviceSelect: ((id: string) => void) | undefined` Callback called whenever the user or application selects a new audio output. +- `controls.setAudioDevice(id: string): void` Sets the selected audio device in Element Call's menu. This should be used if the OS decides to automatically switch to Bluetooth, for example. +- `controls.setAudioEnabled(enabled: boolean)` Enables/disables all audio output from the application. Output is enabled by default. +- `showNativeAudioDevicePicker: (() => void) | undefined`. Callback called whenever the user presses the output button in the settings menu. + This button is only shown on iOS. (`userAgent.includes("iPhone")`) diff --git a/docs/url-params.md b/docs/url-params.md index c533937bd..09525ee20 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -63,6 +63,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `lang` | [BCP 47](https://www.rfc-editor.org/info/bcp47) code | No | No | The language the app should use. | | `password` | | No | No | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.) | | `perParticipantE2EE` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Enables per participant encryption with Keys exchanged over encrypted matrix room messages. | +| `controlledAudioDevices` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the [global JS controls for audio output devices](./controls.md#audio-devices) should be enabled, allowing the list of audio devices to be controlled by the app hosting Element Call. | | `roomId` | [Matrix Room ID](https://spec.matrix.org/v1.12/appendices/#room-ids) | Yes | No | Anything about what room we're pointed to should be from useRoomIdentifier which parses the path and resolves alias with respect to the default server name, however roomId is an exception as we need the room ID in embedded widget mode, and not the room alias (or even the via params because we are not trying to join it). This is also not validated, where it is in `useRoomIdentifier()`. | | `showControls` | `true` or `false` | No, defaults to `true` | No, defaults to `true` | Displays controls like mute, screen-share, invite, and hangup buttons during a call. | | `skipLobby` (deprecated: use `intent` instead) | `true` or `false` | No. If `intent` is explicitly `start_call` then defaults to `true`. Otherwise defaults to `false` | No, defaults to `false` | Skips the lobby to join a call directly, can be combined with preload in widget. When `true` the audio and video inputs will be muted by default. (This means there currently is no way to start without muted video if one wants to skip the lobby. Also not in widget mode.) | diff --git a/locales/en/app.json b/locales/en/app.json index 0b4c05992..e8a86fcc2 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -173,6 +173,7 @@ "devices": { "camera": "Camera", "camera_numbered": "Camera {{n}}", + "change_device_button": "Change audio device", "default": "Default", "default_named": "Default <2>({{name}})", "earpiece": "Earpiece", diff --git a/package.json b/package.json index 5d5bce55b..970b641be 100644 --- a/package.json +++ b/package.json @@ -99,7 +99,7 @@ "i18next-parser": "^9.1.0", "jsdom": "^26.0.0", "knip": "^5.27.2", - "livekit-client": "^2.11.3", + "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/add-room-key-fallback-on-encryption-manager-not-supported", diff --git a/src/UrlParams.ts b/src/UrlParams.ts index fce95445f..17e169d9e 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -124,9 +124,15 @@ export interface UrlParams { */ password: string | null; /** - * Whether we the app should use per participant keys for E2EE. + * Whether the app should use per participant keys for E2EE. */ perParticipantE2EE: boolean; + /** + * Whether the global JS controls for audio output devices should be enabled, + * allowing the list of output devices to be controlled by the app hosting + * Element Call. + */ + controlledAudioDevices: boolean; /** * Setting this flag skips the lobby and brings you in the call directly. * In the widget this can be combined with preload to pass the device settings @@ -173,6 +179,7 @@ export interface UrlParams { * The Sentry DSN. This is only used in the embedded package of Element Call. */ sentryDsn: string | null; + /** * The Sentry environment. This is only used in the embedded package of Element Call. */ @@ -281,6 +288,11 @@ export const getUrlParams = ( fontScale: Number.isNaN(fontScale) ? null : fontScale, allowIceFallback: parser.getFlagParam("allowIceFallback"), perParticipantE2EE: parser.getFlagParam("perParticipantE2EE"), + controlledAudioDevices: parser.getFlagParam( + "controlledAudioDevices", + // the deprecated property name + parser.getFlagParam("controlledMediaDevices"), + ), skipLobby: parser.getFlagParam( "skipLobby", isWidget && intent === UserIntent.StartNewCall, diff --git a/src/controls.ts b/src/controls.ts index b708c9beb..320f41ca5 100644 --- a/src/controls.ts +++ b/src/controls.ts @@ -5,15 +5,55 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Subject } from "rxjs"; +import { BehaviorSubject, Subject } from "rxjs"; export interface Controls { - canEnterPip: () => boolean; - enablePip: () => void; - disablePip: () => void; + canEnterPip(): boolean; + enablePip(): void; + disablePip(): void; + /** @deprecated use setAvailableAudioDevices instead*/ + setAvailableOutputDevices(devices: OutputDevice[]): void; + setAvailableAudioDevices(devices: OutputDevice[]): void; + /** @deprecated use setAudioDevice instead*/ + setOutputDevice(id: string): void; + setAudioDevice(id: string): void; + /** @deprecated use onAudioDeviceSelect instead*/ + onOutputDeviceSelect?: (id: string) => void; + onAudioDeviceSelect?: (id: string) => void; + /** @deprecated use setAudioEnabled instead*/ + setOutputEnabled(enabled: boolean): void; + setAudioEnabled(enabled: boolean): void; + /** @deprecated use showNativeAudioDevicePicker instead*/ + showNativeOutputDevicePicker?: () => void; + showNativeAudioDevicePicker?: () => void; } +export interface OutputDevice { + id: string; + name: string; + forEarpiece?: boolean; + isEarpiece?: boolean; + isSpeaker?: boolean; + isExternalHeadset?: boolean; +} + +/** + * If pipMode is enabled, EC will render a adapted call view layout. + */ export const setPipEnabled$ = new Subject(); +// BehaviorSubject since the client might set this before we have subscribed (GroupCallView still in "loading" state) +// We want the devices that have been set during loading to be available immediately once loaded. +export const availableOutputDevices$ = new BehaviorSubject([]); +// BehaviorSubject since the client might set this before we have subscribed (GroupCallView still in "loading" state) +// We want the device that has been set during loading to be available immediately once loaded. +export const outputDevice$ = new BehaviorSubject(undefined); +/** + * This allows the os to mute the call if the user + * presses the volume down button when it is at the minimum volume. + * + * This should also be used to display a darkened overlay screen letting the user know that audio is muted. + */ +export const setAudioEnabled$ = new Subject(); window.controls = { canEnterPip(): boolean { @@ -27,4 +67,28 @@ window.controls = { if (!setPipEnabled$.observed) throw new Error("No call is running"); setPipEnabled$.next(false); }, + setAvailableAudioDevices(devices: OutputDevice[]): void { + availableOutputDevices$.next(devices); + }, + setAudioDevice(id: string): void { + outputDevice$.next(id); + }, + setAudioEnabled(enabled: boolean): void { + if (!setAudioEnabled$.observed) + throw new Error( + "Output controls are disabled. No setAudioEnabled$ observer", + ); + setAudioEnabled$.next(enabled); + }, + + // wrappers for the deprecated controls fields + setOutputEnabled(enabled: boolean): void { + this.setAudioEnabled(enabled); + }, + setAvailableOutputDevices(devices: OutputDevice[]): void { + this.setAvailableAudioDevices(devices); + }, + setOutputDevice(id: string): void { + this.setAudioDevice(id); + }, }; diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index 7d82032ac..636f5a6db 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -1,5 +1,5 @@ /* -Copyright 2023, 2024 New Vector Ltd. +Copyright 2023-2025 New Vector Ltd. SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. @@ -17,8 +17,8 @@ import { type JSX, } from "react"; import { createMediaDeviceObserver } from "@livekit/components-core"; -import { map, startWith } from "rxjs"; -import { useObservableEagerState } from "observable-hooks"; +import { combineLatest, map, startWith } from "rxjs"; +import { useObservable, useObservableEagerState } from "observable-hooks"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -29,7 +29,11 @@ import { alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting, type Setting, } from "../settings/settings"; +import { outputDevice$, availableOutputDevices$ } from "../controls"; +import { useUrlParams } from "../UrlParams"; +// This hardcoded id is used in EX ios! It can only be changed in coordination with +// the ios swift team. export const EARPIECE_CONFIG_ID = "earpiece-id"; export type DeviceLabel = @@ -38,7 +42,7 @@ export type DeviceLabel = | { type: "earpiece" } | { type: "default"; name: string | null }; -export interface MediaDevice { +export interface MediaDeviceHandle { /** * A map from available device IDs to labels. */ @@ -59,24 +63,69 @@ export interface MediaDevice { select: (deviceId: string) => void; } -export interface MediaDevices { - audioInput: MediaDevice; - audioOutput: MediaDevice; - videoInput: MediaDevice; +interface InputDevices { + audioInput: MediaDeviceHandle; + videoInput: MediaDeviceHandle; startUsingDeviceNames: () => void; stopUsingDeviceNames: () => void; + usingNames: boolean; } -function useMediaDevice( +export interface MediaDevices extends Omit { + audioOutput: MediaDeviceHandle; +} + +/** + * An observable that represents if we should display the devices menu for iOS. + * This implies the following + * - hide any input devices (they do not work anyhow on ios) + * - Show a button to show the native output picker instead. + * - Only show the earpiece toggle option if the earpiece is available: + * `availableOutputDevices$.includes((d)=>d.forEarpiece)` + */ +export const iosDeviceMenu$ = alwaysShowIphoneEarpieceSetting.value$.pipe( + map((v) => v || navigator.userAgent.includes("iPhone")), +); + +function useSelectedId( + available: Map, + preferredId: string | undefined, +): string | undefined { + return useMemo(() => { + if (available.size) { + // If the preferred device is available, use it. Or if every available + // device ID is falsy, the browser is probably just being paranoid about + // fingerprinting and we should still try using the preferred device. + // Worst case it is not available and the browser will gracefully fall + // back to some other device for us when requesting the media stream. + // Otherwise, select the first available device. + return (preferredId !== undefined && available.has(preferredId)) || + (available.size === 1 && available.has("")) + ? preferredId + : available.keys().next().value; + } + return undefined; + }, [available, preferredId]); +} + +/** + * Hook to get access to a mediaDevice handle for a kind. This allows to list + * the available devices, read and set the selected device. + * @param kind Audio input, output or video output. + * @param setting The setting this handle's selection should be synced with. + * @param usingNames If the hook should query device names for the associated + * list. + * @returns A handle for the chosen kind. + */ +function useMediaDeviceHandle( kind: MediaDeviceKind, setting: Setting, usingNames: boolean, -): MediaDevice { - // Make sure we don't needlessly reset to a device observer without names, - // once permissions are already given - const [alwaysShowIphoneEarpice] = useSetting(alwaysShowIphoneEarpieceSetting); +): MediaDeviceHandle { const hasRequestedPermissions = useRef(false); const requestPermissions = usingNames || hasRequestedPermissions.current; + // Make sure we don't needlessly reset to a device observer without names, + // once permissions are already given hasRequestedPermissions.current ||= usingNames; // We use a bare device observer here rather than one of the fancy device @@ -114,52 +163,28 @@ function useMediaDevice( // recognizes. // We also create this if we do not have any available devices, so that // we can use the default or the earpiece. - const showEarpiece = - navigator.userAgent.match("iPhone") || alwaysShowIphoneEarpice; if ( kind === "audiooutput" && !available.has("") && !available.has("default") && - (available.size || showEarpiece) + available.size ) available = new Map([ ["", { type: "default", name: availableRaw[0]?.label || null }], ...available, ]); - if (kind === "audiooutput" && showEarpiece) - // On IPhones we have to create a virtual earpiece device, because - // the earpiece is not available as a device ID. - available = new Map([ - ...available, - [EARPIECE_CONFIG_ID, { type: "earpiece" }], - ]); // Note: creating virtual default input devices would be another problem // entirely, because requesting a media stream from deviceId "" won't // automatically track the default device. return available; }), ), - [alwaysShowIphoneEarpice, deviceObserver$, kind], + [deviceObserver$, kind], ), ); - const [preferredId, setPreferredId] = useSetting(setting); - const [asEarpice, setAsEarpiece] = useState(false); - const selectedId = useMemo(() => { - if (available.size) { - // If the preferred device is available, use it. Or if every available - // device ID is falsy, the browser is probably just being paranoid about - // fingerprinting and we should still try using the preferred device. - // Worst case it is not available and the browser will gracefully fall - // back to some other device for us when requesting the media stream. - // Otherwise, select the first available device. - return (preferredId !== undefined && available.has(preferredId)) || - (available.size === 1 && available.has("")) - ? preferredId - : available.keys().next().value; - } - return undefined; - }, [available, preferredId]); + const [preferredId, select] = useSetting(setting); + const selectedId = useSelectedId(available, preferredId); const selectedGroupId = useObservableEagerState( useMemo( @@ -174,37 +199,26 @@ function useMediaDevice( ), ); - const select = useCallback( - (id: string) => { - if (id === EARPIECE_CONFIG_ID) { - setAsEarpiece(true); - } else { - setAsEarpiece(false); - setPreferredId(id); - } - }, - [setPreferredId], - ); - return useMemo( () => ({ available, selectedId, - useAsEarpiece: asEarpice, + useAsEarpiece: false, selectedGroupId, select, }), - [available, selectedId, asEarpice, selectedGroupId, select], + [available, selectedId, selectedGroupId, select], ); } -export const deviceStub: MediaDevice = { +export const deviceStub: MediaDeviceHandle = { available: new Map(), selectedId: undefined, selectedGroupId: undefined, select: () => {}, useAsEarpiece: false, }; + export const devicesStub: MediaDevices = { audioInput: deviceStub, audioOutput: deviceStub, @@ -215,26 +229,17 @@ export const devicesStub: MediaDevices = { export const MediaDevicesContext = createContext(devicesStub); -interface Props { - children: JSX.Element; -} - -export const MediaDevicesProvider: FC = ({ children }) => { +function useInputDevices(): InputDevices { // Counts the number of callers currently using device names. const [numCallersUsingNames, setNumCallersUsingNames] = useState(0); const usingNames = numCallersUsingNames > 0; - const audioInput = useMediaDevice( + const audioInput = useMediaDeviceHandle( "audioinput", audioInputSetting, usingNames, ); - const audioOutput = useMediaDevice( - "audiooutput", - audioOutputSetting, - usingNames, - ); - const videoInput = useMediaDevice( + const videoInput = useMediaDeviceHandle( "videoinput", videoInputSetting, usingNames, @@ -249,17 +254,52 @@ export const MediaDevicesProvider: FC = ({ children }) => { [setNumCallersUsingNames], ); + return { + audioInput, + videoInput, + startUsingDeviceNames, + stopUsingDeviceNames, + usingNames, + }; +} + +interface Props { + children: JSX.Element; +} + +export const MediaDevicesProvider: FC = ({ children }) => { + const { + audioInput, + videoInput, + startUsingDeviceNames, + stopUsingDeviceNames, + usingNames, + } = useInputDevices(); + + const { controlledAudioDevices } = useUrlParams(); + + const webViewAudioOutput = useMediaDeviceHandle( + "audiooutput", + audioOutputSetting, + usingNames, + ); + const controlledAudioOutput = useControlledOutput(); + const context: MediaDevices = useMemo( () => ({ audioInput, - audioOutput, + audioOutput: controlledAudioDevices + ? controlledAudioOutput + : webViewAudioOutput, videoInput, startUsingDeviceNames, stopUsingDeviceNames, }), [ audioInput, - audioOutput, + controlledAudioDevices, + controlledAudioOutput, + webViewAudioOutput, videoInput, startUsingDeviceNames, stopUsingDeviceNames, @@ -273,6 +313,80 @@ export const MediaDevicesProvider: FC = ({ children }) => { ); }; +function useControlledOutput(): MediaDeviceHandle { + const { available } = useObservableEagerState( + useObservable(() => { + const outputDeviceData$ = availableOutputDevices$.pipe( + map((devices) => { + const deviceForEarpiece = devices.find((d) => d.forEarpiece); + const deviceMapTuple: [string, DeviceLabel][] = devices.map( + ({ id, name, isEarpiece, isSpeaker /*,isExternalHeadset*/ }) => { + let deviceLabel: DeviceLabel = { type: "name", name }; + // if (isExternalHeadset) // Do we want this? + if (isEarpiece) deviceLabel = { type: "earpiece" }; + if (isSpeaker) deviceLabel = { type: "default", name }; + return [id, deviceLabel]; + }, + ); + return { + devicesMap: new Map(deviceMapTuple), + deviceForEarpiece, + }; + }), + ); + + return combineLatest( + [outputDeviceData$, iosDeviceMenu$], + ({ devicesMap, deviceForEarpiece }, iosShowEarpiece) => { + let available = devicesMap; + if (iosShowEarpiece && !!deviceForEarpiece) { + available = new Map([ + ...devicesMap.entries(), + [EARPIECE_CONFIG_ID, { type: "earpiece" }], + ]); + } + return { available, deviceForEarpiece }; + }, + ); + }), + ); + const [preferredId, setPreferredId] = useSetting(audioOutputSetting); + useEffect(() => { + const subscription = outputDevice$.subscribe((id) => { + if (id) setPreferredId(id); + }); + return (): void => subscription.unsubscribe(); + }, [setPreferredId]); + + const selectedId = useSelectedId(available, preferredId); + + const [asEarpiece, setAsEarpiece] = useState(false); + + useEffect(() => { + // Let the hosting application know which output device has been selected. + // This information is probably only of interest if the earpiece mode has been + // selected - for example, Element X iOS listens to this to determine whether it + // should enable the proximity sensor. + if (selectedId) { + window.controls.onAudioDeviceSelect?.(selectedId); + // Call deprecated method for backwards compatibility. + window.controls.onOutputDeviceSelect?.(selectedId); + } + setAsEarpiece(selectedId === EARPIECE_CONFIG_ID); + }, [selectedId]); + + return useMemo( + () => ({ + available: available, + selectedId, + selectedGroupId: undefined, + select: setPreferredId, + useAsEarpiece: asEarpiece, + }), + [available, selectedId, setPreferredId, asEarpiece], + ); +} + export const useMediaDevices = (): MediaDevices => useContext(MediaDevicesContext); diff --git a/src/livekit/useLivekit.ts b/src/livekit/useLivekit.ts index 7cb32f5f8..53f366d2b 100644 --- a/src/livekit/useLivekit.ts +++ b/src/livekit/useLivekit.ts @@ -25,7 +25,7 @@ import { defaultLiveKitOptions } from "./options"; import { type SFUConfig } from "./openIDSFU"; import { type MuteStates } from "../room/MuteStates"; import { - type MediaDevice, + type MediaDeviceHandle, type MediaDevices, useMediaDevices, } from "./MediaDevicesContext"; @@ -304,7 +304,10 @@ export function useLivekit( useEffect(() => { // Sync the requested devices with LiveKit's devices if (room !== undefined && connectionState === ConnectionState.Connected) { - const syncDevice = (kind: MediaDeviceKind, device: MediaDevice): void => { + const syncDevice = ( + kind: MediaDeviceKind, + device: MediaDeviceHandle, + ): void => { const id = device.selectedId; // Detect if we're trying to use chrome's default device, in which case diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index f1027b5ce..4097af6c8 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -24,6 +24,7 @@ import { type MatrixRTCSession, } from "matrix-js-sdk/lib/matrixrtc"; import { useNavigate } from "react-router-dom"; +import { useObservableEagerState } from "observable-hooks"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { @@ -64,10 +65,10 @@ import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx"; import { useNewMembershipManager as useNewMembershipManagerSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, - muteAllAudio as muteAllAudioSetting, useSetting, } from "../settings/settings"; import { useTypedEventEmitter } from "../useEvents"; +import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; declare global { interface Window { @@ -104,9 +105,9 @@ export const GroupCallView: FC = ({ const [externalError, setExternalError] = useState( null, ); - - const [muteAllAudio] = useSetting(muteAllAudioSetting); const memberships = useMatrixRTCSessionMemberships(rtcSession); + + const muteAllAudio = useObservableEagerState(muteAllAudio$); const leaveSoundContext = useLatest( useAudioContext({ sounds: callEventAudioSounds, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 829d9c68c..b9655d377 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -96,7 +96,6 @@ import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { debugTileLayout as debugTileLayoutSetting, useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting, - muteAllAudio as muteAllAudioSetting, developerMode as developerModeSetting, useSetting, } from "../settings/settings"; @@ -104,6 +103,7 @@ import { ReactionsReader } from "../reactions/ReactionsReader"; import { ConnectionLostError } from "../utils/errors.ts"; import { useTypedEventEmitter } from "../useEvents.ts"; import { MatrixAudioRenderer } from "../livekit/MatrixAudioRenderer.tsx"; +import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -235,7 +235,7 @@ export const InCallView: FC = ({ room: livekitRoom, }); - const [muteAllAudio] = useSetting(muteAllAudioSetting); + const muteAllAudio = useObservableEagerState(muteAllAudio$); // This seems like it might be enough logic to use move it into the call view model? const [didFallbackToRoomKey, setDidFallbackToRoomKey] = useState(false); diff --git a/src/room/MuteStates.test.tsx b/src/room/MuteStates.test.tsx index eb0666038..65e7d3331 100644 --- a/src/room/MuteStates.test.tsx +++ b/src/room/MuteStates.test.tsx @@ -14,7 +14,7 @@ import userEvent from "@testing-library/user-event"; import { useMuteStates } from "./MuteStates"; import { type DeviceLabel, - type MediaDevice, + type MediaDeviceHandle, type MediaDevices, MediaDevicesContext, } from "../livekit/MediaDevicesContext"; @@ -73,7 +73,7 @@ const mockCamera: MediaDeviceInfo = { }, }; -function mockDevices(available: Map): MediaDevice { +function mockDevices(available: Map): MediaDeviceHandle { return { available, selectedId: "", diff --git a/src/room/MuteStates.ts b/src/room/MuteStates.ts index e57ba7d52..6e24fb07a 100644 --- a/src/room/MuteStates.ts +++ b/src/room/MuteStates.ts @@ -16,7 +16,7 @@ import { type IWidgetApiRequest } from "matrix-widget-api"; import { logger } from "matrix-js-sdk/lib/logger"; import { - type MediaDevice, + type MediaDeviceHandle, useMediaDevices, } from "../livekit/MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; @@ -53,7 +53,7 @@ export interface MuteStates { } function useMuteState( - device: MediaDevice, + device: MediaDeviceHandle, enabledByDefault: () => boolean, ): MuteState { const [enabled, setEnabled] = useReactiveState( diff --git a/src/settings/DeviceSelection.tsx b/src/settings/DeviceSelection.tsx index 396b1235c..aee043c63 100644 --- a/src/settings/DeviceSelection.tsx +++ b/src/settings/DeviceSelection.tsx @@ -24,12 +24,12 @@ import { Trans, useTranslation } from "react-i18next"; import { EARPIECE_CONFIG_ID, - type MediaDevice, + type MediaDeviceHandle, } from "../livekit/MediaDevicesContext"; import styles from "./DeviceSelection.module.css"; interface Props { - device: MediaDevice; + device: MediaDeviceHandle; title: string; numberedLabel: (number: number) => string; } diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 1c97a87d5..57463fc7e 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -8,8 +8,9 @@ Please see LICENSE in the repository root for full details. import { type FC, type ReactNode, useState } from "react"; import { useTranslation } from "react-i18next"; import { type MatrixClient } from "matrix-js-sdk"; -import { Root as Form, Separator } from "@vector-im/compound-web"; +import { Button, Root as Form, Separator } from "@vector-im/compound-web"; import { type Room as LivekitRoom } from "livekit-client"; +import { useObservableEagerState } from "observable-hooks"; import { Modal } from "../Modal"; import styles from "./SettingsModal.module.css"; @@ -19,6 +20,7 @@ import { FeedbackSettingsTab } from "./FeedbackSettingsTab"; import { useMediaDevices, useMediaDeviceNames, + iosDeviceMenu$, } from "../livekit/MediaDevicesContext"; import { widget } from "../widget"; import { @@ -34,6 +36,7 @@ import { useTrackProcessor } from "../livekit/TrackProcessorContext"; import { DeveloperSettingsTab } from "./DeveloperSettingsTab"; import { FieldRow, InputField } from "../input/Input"; import { useSubmitRageshake } from "./submit-rageshake"; +import { useUrlParams } from "../UrlParams"; type SettingsTab = | "audio" @@ -102,19 +105,42 @@ export const SettingsModal: FC = ({ const { available: isRageshakeAvailable } = useSubmitRageshake(); + // For controlled devices, we will not show the input section: + // Controlled media devices are used on mobile platforms, where input and output are grouped into + // a single device. These are called "headset" or "speaker" (or similar) but contain both input and output. + // On EC, we decided that it is less confusing for the user if they see those options in the output section + // rather than the input section. + const { controlledAudioDevices } = useUrlParams(); + // If we are on iOS we will show a button to open the native audio device picker. + const iosDeviceMenu = useObservableEagerState(iosDeviceMenu$); + const audioTab: Tab = { key: "audio", name: t("common.audio"), content: ( <>
- - t("settings.devices.microphone_numbered", { n }) - } - /> + {!controlledAudioDevices && ( + + t("settings.devices.microphone_numbered", { n }) + } + /> + )} + {iosDeviceMenu && ( + + )} ("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); export const alwaysShowIphoneEarpiece = new Setting( - "always-show-iphone-earpice", + "always-show-iphone-earpiece", false, ); diff --git a/src/state/MuteAllAudioModel.test.ts b/src/state/MuteAllAudioModel.test.ts new file mode 100644 index 000000000..d7b4e0c42 --- /dev/null +++ b/src/state/MuteAllAudioModel.test.ts @@ -0,0 +1,36 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { test, vi } from "vitest"; +import { expect } from "vitest"; + +import { setAudioEnabled$ } from "../controls"; +import { muteAllAudio as muteAllAudioSetting } from "../settings/settings"; +import { muteAllAudio$ } from "./MuteAllAudioModel"; + +test("muteAllAudio$", () => { + const valueMock = vi.fn(); + const muteAllAudio = muteAllAudio$.subscribe((value) => { + valueMock(value); + }); + + setAudioEnabled$.next(false); + setAudioEnabled$.next(true); + muteAllAudioSetting.setValue(false); + muteAllAudioSetting.setValue(true); + setAudioEnabled$.next(false); + + muteAllAudio.unsubscribe(); + + expect(valueMock).toHaveBeenCalledTimes(6); + expect(valueMock).toHaveBeenNthCalledWith(1, false); // startWith([false, muteAllAudioSetting.getValue()]); + expect(valueMock).toHaveBeenNthCalledWith(2, true); // setAudioEnabled$.next(false); + expect(valueMock).toHaveBeenNthCalledWith(3, false); // setAudioEnabled$.next(true); + expect(valueMock).toHaveBeenNthCalledWith(4, false); // muteAllAudioSetting.setValue(false); + expect(valueMock).toHaveBeenNthCalledWith(5, true); // muteAllAudioSetting.setValue(true); + expect(valueMock).toHaveBeenNthCalledWith(6, true); // setAudioEnabled$.next(false); +}); diff --git a/src/state/MuteAllAudioModel.ts b/src/state/MuteAllAudioModel.ts new file mode 100644 index 000000000..16f4a0ecc --- /dev/null +++ b/src/state/MuteAllAudioModel.ts @@ -0,0 +1,19 @@ +/* +Copyright 2025 New Vector Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { combineLatest, startWith } from "rxjs"; + +import { setAudioEnabled$ } from "../controls"; +import { muteAllAudio as muteAllAudioSetting } from "../settings/settings"; + +/** + * This can transition into sth more complete: `GroupCallViewModel.ts` + */ +export const muteAllAudio$ = combineLatest( + [setAudioEnabled$.pipe(startWith(true)), muteAllAudioSetting.value$], + (outputEnabled, settingsMute) => !outputEnabled || settingsMute, +); diff --git a/src/useAudioContext.test.tsx b/src/useAudioContext.test.tsx index dd3c3b0cd..f2e2efdba 100644 --- a/src/useAudioContext.test.tsx +++ b/src/useAudioContext.test.tsx @@ -140,7 +140,7 @@ test("will use the correct volume level", async () => { expect(testAudioContext.pan.pan.setValueAtTime).toHaveBeenCalledWith(0, 0); }); -test("will use the pan if earpice is selected", async () => { +test("will use the pan if earpiece is selected", async () => { const { findByText } = render( Date: Thu, 22 May 2025 19:25:52 +0200 Subject: [PATCH 2/3] Bump-js-sdk: switch to existing js-sdk version --- package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 970b641be..97c4f737f 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "livekit-client": "^2.13.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/add-room-key-fallback-on-encryption-manager-not-supported", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop", "matrix-widget-api": "^1.13.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 61e093b0b..ea7324c3e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2547,10 +2547,10 @@ __metadata: languageName: node linkType: hard -"@matrix-org/matrix-sdk-crypto-wasm@npm:^14.0.1": - version: 14.0.1 - resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:14.0.1" - checksum: 10c0/6e98abb61f8d6c43b26f04e83db92b39db74352861495eda9ac472b2f58411a45b2f150e4361c44c6800f98f99e89350d11941e87b9bf22204c9cab83ca93e27 +"@matrix-org/matrix-sdk-crypto-wasm@npm:^14.2.0": + version: 14.2.0 + resolution: "@matrix-org/matrix-sdk-crypto-wasm@npm:14.2.0" + checksum: 10c0/cc51417d71ffe506401dbb85ff4930bf89b33b6718c9052ba3cc1b2989e989e5e7e6fceea7c5fb58576d64e7a5c947c4cece89dbfa785083d293508c80e0713d languageName: node linkType: hard @@ -7007,7 +7007,7 @@ __metadata: livekit-client: "npm:^2.13.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/add-room-key-fallback-on-encryption-manager-not-supported" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop" matrix-widget-api: "npm:^1.13.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -9610,12 +9610,12 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/add-room-key-fallback-on-encryption-manager-not-supported": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop": version: 37.6.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=6b3cf9667f306a79d57bfe9688d3414c0a8552a8" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=93982716951ce2583904bfc26b27d6a86ba17a87" dependencies: "@babel/runtime": "npm:^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.0.1" + "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.2.0" "@matrix-org/olm": "npm:3.2.15" another-json: "npm:^0.2.0" bs58: "npm:^6.0.0" @@ -9629,7 +9629,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/be747a85458764c2e1fe970a3f7ea00de24c7e20d12a125d038f730a91e3dc49926290ad4dd840d063233c035f8b3ed19fe551185376f21e98cf782fe25690a3 + checksum: 10c0/22a28099d2deaf0ca7f609a5859fe00fbd20c314d3b607a95b4a623a8aa159e428310b1849c77ab7b9875a29fc2d924cddbb6fc83dc4e42a74c4713401dcb0be languageName: node linkType: hard From 5093ab271764ac9291871af24388e43d07d2a2b8 Mon Sep 17 00:00:00 2001 From: Timo Date: Thu, 22 May 2025 19:35:56 +0200 Subject: [PATCH 3/3] lint --- docs/url-params.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/url-params.md b/docs/url-params.md index 09525ee20..88807777b 100644 --- a/docs/url-params.md +++ b/docs/url-params.md @@ -63,7 +63,7 @@ These parameters are relevant to both [widget](./embedded-standalone.md) and [st | `lang` | [BCP 47](https://www.rfc-editor.org/info/bcp47) code | No | No | The language the app should use. | | `password` | | No | No | E2EE password when using a shared secret. (For individual sender keys in embedded mode this is not required.) | | `perParticipantE2EE` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Enables per participant encryption with Keys exchanged over encrypted matrix room messages. | -| `controlledAudioDevices` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the [global JS controls for audio output devices](./controls.md#audio-devices) should be enabled, allowing the list of audio devices to be controlled by the app hosting Element Call. | +| `controlledAudioDevices` | `true` or `false` | No, defaults to `false` | No, defaults to `false` | Whether the [global JS controls for audio output devices](./controls.md#audio-devices) should be enabled, allowing the list of audio devices to be controlled by the app hosting Element Call. | | `roomId` | [Matrix Room ID](https://spec.matrix.org/v1.12/appendices/#room-ids) | Yes | No | Anything about what room we're pointed to should be from useRoomIdentifier which parses the path and resolves alias with respect to the default server name, however roomId is an exception as we need the room ID in embedded widget mode, and not the room alias (or even the via params because we are not trying to join it). This is also not validated, where it is in `useRoomIdentifier()`. | | `showControls` | `true` or `false` | No, defaults to `true` | No, defaults to `true` | Displays controls like mute, screen-share, invite, and hangup buttons during a call. | | `skipLobby` (deprecated: use `intent` instead) | `true` or `false` | No. If `intent` is explicitly `start_call` then defaults to `true`. Otherwise defaults to `false` | No, defaults to `false` | Skips the lobby to join a call directly, can be combined with preload in widget. When `true` the audio and video inputs will be muted by default. (This means there currently is no way to start without muted video if one wants to skip the lobby. Also not in widget mode.) |