diff --git a/package.json b/package.json index 6ff4198da..1c17d42eb 100644 --- a/package.json +++ b/package.json @@ -99,10 +99,10 @@ "i18next-parser": "^9.1.0", "jsdom": "^26.0.0", "knip": "^5.27.2", - "livekit-client": "^2.11.3", + "livekit-client": "^2.12.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#19b1b901f575755d29d1fe03ca48cbf7c1cae05c", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#ba2a87cc204c2b33abd63735936b281b07692815", "matrix-widget-api": "1.11.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", @@ -133,6 +133,7 @@ }, "resolutions": { "@livekit/components-core/rxjs": "^7.8.1", + "loglevel": "1.9.2", "@livekit/track-processors/@mediapipe/tasks-vision": "^0.10.18", "matrix-widget-api": "1.11.0" }, diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 9b190ed86..cbef32283 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -5,18 +5,51 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { BaseKeyProvider, createKeyMaterialFromBuffer } from "livekit-client"; +import { + BaseKeyProvider, + importKey, + KeyProviderEvent, + type RatchetResult, +} from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; +import { encodeBase64 } from "matrix-js-sdk"; + +function tmpLogPrefix(msg: string): void { + logger.debug(`[RTCEncryption][MatrixKeyProvider] - ${msg}`); +} export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; + private readonly onKeyRatchetComplete: ( + ratchetResult: RatchetResult, + participantIdentity?: string, + keyIndex?: number, + ) => void; + public constructor() { - super({ ratchetWindowSize: 10, keyringSize: 256 }); + super({ ratchetWindowSize: 10, keyringSize: 10 }); + + this.onKeyRatchetComplete = ( + ratchetResult: RatchetResult, + participantIdentity?: string, + keyIndex?: number, + ): void => { + tmpLogPrefix( + `key ratcheted event received for ${participantIdentity} at index ${keyIndex} key=${encodeBase64(new Uint8Array(ratchetResult.chainKey))}`, + ); + this.rtcSession?.onKeyRatcheted( + ratchetResult.chainKey, + participantIdentity, + keyIndex, + ); + }; + + this.on(KeyProviderEvent.KeyRatcheted, this.onKeyRatchetComplete); } public setRTCSession(rtcSession: MatrixRTCSession): void { @@ -25,6 +58,10 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); + this.rtcSession.off( + MatrixRTCSessionEvent.EncryptionKeyQueryRatchetStep, + this.doRatchetKey, + ); } this.rtcSession = rtcSession; @@ -34,21 +71,36 @@ export class MatrixKeyProvider extends BaseKeyProvider { this.onEncryptionKeyChanged, ); + this.rtcSession.on( + MatrixRTCSessionEvent.EncryptionKeyQueryRatchetStep, + this.doRatchetKey, + ); + tmpLogPrefix(`setRTCSession SESSION CHANGED`); // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } + private doRatchetKey = (participantId: string, keyIndex: number): void => { + tmpLogPrefix( + `doRatchetKey participantId=${participantId} keyIndex=${keyIndex}`, + ); + this.ratchetKey(participantId, keyIndex); + }; + private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, participantId: string, ): void => { - createKeyMaterialFromBuffer(encryptionKey).then( + tmpLogPrefix( + `onEncryptionKeyChanged participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex} key=${encodeBase64(encryptionKey)}`, + ); + importKey(encryptionKey, "HKDF", "derive").then( (keyMaterial) => { this.onSetEncryptionKey(keyMaterial, participantId, encryptionKeyIndex); - logger.debug( + tmpLogPrefix( `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`, ); }, diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 768ddfdd7..48e2f1a1b 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -139,6 +139,7 @@ export const ActiveCall: FC = (props) => { }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + const [showDeveloperModeDebugOptions] = useSetting(developerModeSetting); useEffect(() => { if (livekitRoom !== undefined) { @@ -150,6 +151,7 @@ export const ActiveCall: FC = (props) => { connStateObservable$, reactionsReader.raisedHands$, reactionsReader.reactions$, + showDeveloperModeDebugOptions, ); setVm(vm); return (): void => { @@ -157,7 +159,13 @@ export const ActiveCall: FC = (props) => { reactionsReader.destroy(); }; } - }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable$]); + }, [ + props.rtcSession, + livekitRoom, + props.e2eeSystem, + connStateObservable$, + showDeveloperModeDebugOptions, + ]); if (livekitRoom === undefined || vm === null) return null; diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index ecfd44f74..e5763bb91 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -113,6 +113,7 @@ test("It joins the correct Session", async () => { useLegacyMemberEvents: false, useNewMembershipManager: true, useExperimentalToDeviceTransport: false, + useKeyDelay: 5000, }, ); }); diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 51bc79b8d..875033356 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -126,6 +126,7 @@ export async function enterRTCSession( membershipKeepAlivePeriod: matrixRtcSessionConfig?.membership_keep_alive_period, makeKeyDelay: matrixRtcSessionConfig?.key_rotation_on_leave_delay, + useKeyDelay: 5_000, useExperimentalToDeviceTransport, }, ); diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index f0066476d..f60dec442 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -268,6 +268,7 @@ function withCallViewModel( connectionState$, raisedHands$, new BehaviorSubject({}), + false, ); onTestFinished(() => { diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8fd6f819a..34397eac9 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -251,6 +251,7 @@ class UserMedia { public readonly speaker$: Observable; public readonly presenter$: Observable; + public constructor( public readonly id: string, member: RoomMember | undefined, @@ -487,7 +488,11 @@ export class CallViewModel extends ViewModel { const disambiguate = shouldDisambiguate(member, memberships, room); displaynameMap.set( matrixIdentifier, - calculateDisplayName(member, disambiguate), + this.showDeveloperModeDebugOptions + ? calculateDisplayName(member, disambiguate) + + " | " + + rtcMember.deviceId + : calculateDisplayName(member, disambiguate), ); } return displaynameMap; @@ -1344,6 +1349,7 @@ export class CallViewModel extends ViewModel { private readonly reactionsSubject$: Observable< Record >, + private readonly showDeveloperModeDebugOptions: boolean, ) { super(); } diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index c8a93c735..2c8be7aa6 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -138,6 +138,7 @@ export function getBasicCallViewModelEnvironment( of(ConnectionState.Connected), handRaisedSubject$, reactionsSubject$, + false, ); return { vm, diff --git a/tsconfig.json b/tsconfig.json index be12658ec..07632f206 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "es2022", "module": "es2022", "jsx": "react-jsx", - "lib": ["es2022", "dom", "dom.iterable"], + "lib": ["es2024", "dom", "dom.iterable"], // From Matrix-JS-SDK "strict": true, diff --git a/yarn.lock b/yarn.lock index a42427b4e..1fac138d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6997,10 +6997,10 @@ __metadata: i18next-parser: "npm:^9.1.0" jsdom: "npm:^26.0.0" knip: "npm:^5.27.2" - livekit-client: "npm:^2.11.3" + livekit-client: "npm:^2.12.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#19b1b901f575755d29d1fe03ca48cbf7c1cae05c" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#ba2a87cc204c2b33abd63735936b281b07692815" matrix-widget-api: "npm:1.11.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -9367,7 +9367,7 @@ __metadata: languageName: node linkType: hard -"livekit-client@npm:^2.11.3": +"livekit-client@npm:^2.12.0": version: 2.12.0 resolution: "livekit-client@npm:2.12.0" dependencies: @@ -9430,14 +9430,7 @@ __metadata: languageName: node linkType: hard -"loglevel@npm:1.9.1": - version: 1.9.1 - resolution: "loglevel@npm:1.9.1" - checksum: 10c0/152f0501cea367cf998c844a38b19f0b5af555756ad7d8650214a1f8c6a5b045e31b8cf5dae27d28339a061624ce3f618aadb333aed386cac041d6ddc5101a39 - languageName: node - linkType: hard - -"loglevel@npm:^1.9.1, loglevel@npm:^1.9.2": +"loglevel@npm:1.9.2": version: 1.9.2 resolution: "loglevel@npm:1.9.2" checksum: 10c0/1e317fa4648fe0b4a4cffef6de037340592cee8547b07d4ce97a487abe9153e704b98451100c799b032c72bb89c9366d71c9fb8192ada8703269263ae77acdc7 @@ -9591,9 +9584,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#19b1b901f575755d29d1fe03ca48cbf7c1cae05c": - version: 37.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=19b1b901f575755d29d1fe03ca48cbf7c1cae05c" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#ba2a87cc204c2b33abd63735936b281b07692815": + version: 37.5.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=ba2a87cc204c2b33abd63735936b281b07692815" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.0.1" @@ -9610,7 +9603,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/68a30a113059ba052b2e66502abcd9805f9a18a1bfd1d209203d728b36508af257a57e6248fb237c7018c81bfbe1ec78fa17aea8968c8af0729ea935398dcf8b + checksum: 10c0/bef6ac481a56189aceedb9a784652a0562436cc24aea3e6986c8b47f583f38c8abbad22d55520a6339f690c2625dd12d41dd68533b4ed5bc4e340b8dc7476ab3 languageName: node linkType: hard