From 795230247d4e17197bb3497419eef1baf414bb21 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 29 Apr 2025 15:56:51 +0200 Subject: [PATCH 01/14] WIP ratcheting --- package.json | 4 ++-- src/e2ee/matrixKeyProvider.ts | 35 ++++++++++++++++++++++++++++++++--- 2 files changed, 34 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 6ff4198da..28302e687 100644 --- a/package.json +++ b/package.json @@ -99,10 +99,10 @@ "i18next-parser": "^9.1.0", "jsdom": "^26.0.0", "knip": "^5.27.2", - "livekit-client": "^2.11.3", + "livekit-client": "github:BillCarsonFr/client-sdk-js#4ac197085ee063f66b48d8ec24c88b2321fcbf9e", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#19b1b901f575755d29d1fe03ca48cbf7c1cae05c", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#44d1a860a1c56a0d4193aa7dcd09e10822398a4d", "matrix-widget-api": "1.11.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 9b190ed86..515fec64e 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { BaseKeyProvider, createKeyMaterialFromBuffer } from "livekit-client"; +import { BaseKeyProvider, createKeyMaterialFromBuffer, importKey, KeyProviderEvent } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession, @@ -15,8 +15,22 @@ import { export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; + private readonly onKeyRatchetComplete: (material: ArrayBuffer, keyIndex?: number) => void; + public constructor() { - super({ ratchetWindowSize: 10, keyringSize: 256 }); + super({ ratchetWindowSize: 10, keyringSize: 10 }); + + this.onKeyRatchetComplete = (material: ArrayBuffer, keyIndex?: number): void => { + logger.debug(`key ratcheted event received for index `, keyIndex ); + this.rtcSession?.onOwnKeyRatcheted(material, keyIndex).catch((e) => { + logger.error( + `Failed to ratchet key for livekit room=${this.rtcSession?.room.roomId} keyIndex=${keyIndex}`, + e, + ); + }); + }; + + this.on(KeyProviderEvent.RatchetRequestCompleted, this.onKeyRatchetComplete); } public setRTCSession(rtcSession: MatrixRTCSession): void { @@ -25,6 +39,11 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyChanged, this.onEncryptionKeyChanged, ); + this.rtcSession.off( + MatrixRTCSessionEvent.EncryptionKeyQueryRatchetStep, + this.doRatchetKey, + ); + } this.rtcSession = rtcSession; @@ -34,17 +53,27 @@ export class MatrixKeyProvider extends BaseKeyProvider { this.onEncryptionKeyChanged, ); + this.rtcSession.on( + MatrixRTCSessionEvent.EncryptionKeyQueryRatchetStep, + this.doRatchetKey, + ); + + // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } + private doRatchetKey = (participantId:string, keyIndex:number): void => { + this.ratchetKey(participantId, keyIndex); + } + private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, participantId: string, ): void => { - createKeyMaterialFromBuffer(encryptionKey).then( + importKey(encryptionKey, "HKDF", 'derive').then( (keyMaterial) => { this.onSetEncryptionKey(keyMaterial, participantId, encryptionKeyIndex); From b5dbb626f694e6d7173344a3ffebdfe95c96722b Mon Sep 17 00:00:00 2001 From: Valere Date: Mon, 5 May 2025 14:57:30 +0200 Subject: [PATCH 02/14] Wip ratchet update --- src/e2ee/matrixKeyProvider.ts | 22 +++++++++++----------- yarn.lock | 15 +++++++-------- 2 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 515fec64e..7d35808f7 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -5,7 +5,12 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { BaseKeyProvider, createKeyMaterialFromBuffer, importKey, KeyProviderEvent } from "livekit-client"; +import { + BaseKeyProvider, + importKey, + KeyProviderEvent, + RatchetResult +} from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { type MatrixRTCSession, @@ -15,22 +20,17 @@ import { export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; - private readonly onKeyRatchetComplete: (material: ArrayBuffer, keyIndex?: number) => void; + private readonly onKeyRatchetComplete: (ratchetResult: RatchetResult, participantIdentity?: string, keyIndex?: number) => void; public constructor() { super({ ratchetWindowSize: 10, keyringSize: 10 }); - this.onKeyRatchetComplete = (material: ArrayBuffer, keyIndex?: number): void => { - logger.debug(`key ratcheted event received for index `, keyIndex ); - this.rtcSession?.onOwnKeyRatcheted(material, keyIndex).catch((e) => { - logger.error( - `Failed to ratchet key for livekit room=${this.rtcSession?.room.roomId} keyIndex=${keyIndex}`, - e, - ); - }); + this.onKeyRatchetComplete = (ratchetResult: RatchetResult, participantIdentity?: string, keyIndex?: number): void => { + logger.debug(`key ratcheted event received for ${participantId} at index ${keyIndex}`); + this.rtcSession?.onKeyRatcheted(ratchetResult.chainKey, keyIndex); }; - this.on(KeyProviderEvent.RatchetRequestCompleted, this.onKeyRatchetComplete); + this.on(KeyProviderEvent.KeyRatcheted, this.onKeyRatchetComplete); } public setRTCSession(rtcSession: MatrixRTCSession): void { diff --git a/yarn.lock b/yarn.lock index a42427b4e..8fd32d3fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6997,10 +6997,10 @@ __metadata: i18next-parser: "npm:^9.1.0" jsdom: "npm:^26.0.0" knip: "npm:^5.27.2" - livekit-client: "npm:^2.11.3" + livekit-client: "github:BillCarsonFr/client-sdk-js#4ac197085ee063f66b48d8ec24c88b2321fcbf9e" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#19b1b901f575755d29d1fe03ca48cbf7c1cae05c" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#44d1a860a1c56a0d4193aa7dcd09e10822398a4d" matrix-widget-api: "npm:1.11.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -9382,7 +9382,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" checksum: 10c0/8a4657aa6c0f0bc5d1fe77c2cd9603a3b07d4acefa634f1c5151190eed69711e7e599dd09c07915939a418dc8770d87e3529ecf1b029f2a9af7f2172d83acb1c languageName: node - linkType: hard + linkType: soft "locate-path@npm:^5.0.0": version: 5.0.0 @@ -9591,9 +9591,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#19b1b901f575755d29d1fe03ca48cbf7c1cae05c": - version: 37.4.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=19b1b901f575755d29d1fe03ca48cbf7c1cae05c" +"matrix-js-sdk@portal:/Users/valere/Documents/Dev/matrix/matrix-git-root/matrix-js-sdk::locator=element-call%40workspace%3A.": + version: 0.0.0-use.local + resolution: "matrix-js-sdk@portal:/Users/valere/Documents/Dev/matrix/matrix-git-root/matrix-js-sdk::locator=element-call%40workspace%3A." dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.0.1" @@ -9610,9 +9610,8 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/68a30a113059ba052b2e66502abcd9805f9a18a1bfd1d209203d728b36508af257a57e6248fb237c7018c81bfbe1ec78fa17aea8968c8af0729ea935398dcf8b languageName: node - linkType: hard + linkType: soft "matrix-widget-api@npm:1.11.0": version: 1.11.0 From 8e1a747ac2aa3a093afb7ac31ab7ceba6b7c16f8 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 6 May 2025 11:52:44 +0200 Subject: [PATCH 03/14] fix ratchet event participant id --- src/e2ee/matrixKeyProvider.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 7d35808f7..d3fe4c640 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -26,8 +26,8 @@ export class MatrixKeyProvider extends BaseKeyProvider { super({ ratchetWindowSize: 10, keyringSize: 10 }); this.onKeyRatchetComplete = (ratchetResult: RatchetResult, participantIdentity?: string, keyIndex?: number): void => { - logger.debug(`key ratcheted event received for ${participantId} at index ${keyIndex}`); - this.rtcSession?.onKeyRatcheted(ratchetResult.chainKey, keyIndex); + logger.debug(`key ratcheted event received for ${participantIdentity} at index ${keyIndex}`); + this.rtcSession?.onKeyRatcheted(ratchetResult.chainKey,participantIdentity, keyIndex); }; this.on(KeyProviderEvent.KeyRatcheted, this.onKeyRatchetComplete); From 6bd7fda41104de6727d39cce978e430f6648829e Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 11:13:08 +0200 Subject: [PATCH 04/14] dependency: use rtc with ratchet support --- package.json | 4 ++-- yarn.lock | 17 +++++++++-------- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index 28302e687..e8b53487b 100644 --- a/package.json +++ b/package.json @@ -99,10 +99,10 @@ "i18next-parser": "^9.1.0", "jsdom": "^26.0.0", "knip": "^5.27.2", - "livekit-client": "github:BillCarsonFr/client-sdk-js#4ac197085ee063f66b48d8ec24c88b2321fcbf9e", + "livekit-client": "^2.12.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#44d1a860a1c56a0d4193aa7dcd09e10822398a4d", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f8c27727e34642ef08936d801ce12fa4ef2940c2", "matrix-widget-api": "1.11.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 8fd32d3fd..4d232b4d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6997,10 +6997,10 @@ __metadata: i18next-parser: "npm:^9.1.0" jsdom: "npm:^26.0.0" knip: "npm:^5.27.2" - livekit-client: "github:BillCarsonFr/client-sdk-js#4ac197085ee063f66b48d8ec24c88b2321fcbf9e" + livekit-client: "npm:^2.12.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#44d1a860a1c56a0d4193aa7dcd09e10822398a4d" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#f8c27727e34642ef08936d801ce12fa4ef2940c2" matrix-widget-api: "npm:1.11.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -9367,8 +9367,8 @@ __metadata: languageName: node linkType: hard -"livekit-client@npm:^2.11.3": - version: 2.12.0 +"livekit-client@npm:^2.12.0": + version: 0.0.0-use.local resolution: "livekit-client@npm:2.12.0" dependencies: "@livekit/mutex": "npm:1.1.1" @@ -9591,9 +9591,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@portal:/Users/valere/Documents/Dev/matrix/matrix-git-root/matrix-js-sdk::locator=element-call%40workspace%3A.": - version: 0.0.0-use.local - resolution: "matrix-js-sdk@portal:/Users/valere/Documents/Dev/matrix/matrix-git-root/matrix-js-sdk::locator=element-call%40workspace%3A." +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f8c27727e34642ef08936d801ce12fa4ef2940c2": + version: 37.5.0 + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f8c27727e34642ef08936d801ce12fa4ef2940c2" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.0.1" @@ -9610,8 +9610,9 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" + checksum: 10c0/d05a7491545ff9b68299c763573f378ec09e05437cfd23677a9c97409d4a862ca4d9d73a1adbc624b92a999ba0ddb6d0a105f24cfe9cd669fd42a942a95f25ca languageName: node - linkType: soft + linkType: hard "matrix-widget-api@npm:1.11.0": version: 1.11.0 From 5168946e2fd45867ffc30a6f3d6460a18956d869 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 11:18:03 +0200 Subject: [PATCH 05/14] fixup dep --- package.json | 1 + yarn.lock | 13 +++---------- 2 files changed, 4 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index e8b53487b..dead409ca 100644 --- a/package.json +++ b/package.json @@ -133,6 +133,7 @@ }, "resolutions": { "@livekit/components-core/rxjs": "^7.8.1", + "loglevel": "1.9.2", "@livekit/track-processors/@mediapipe/tasks-vision": "^0.10.18", "matrix-widget-api": "1.11.0" }, diff --git a/yarn.lock b/yarn.lock index 4d232b4d4..1ebb6125d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9368,7 +9368,7 @@ __metadata: linkType: hard "livekit-client@npm:^2.12.0": - version: 0.0.0-use.local + version: 2.12.0 resolution: "livekit-client@npm:2.12.0" dependencies: "@livekit/mutex": "npm:1.1.1" @@ -9382,7 +9382,7 @@ __metadata: webrtc-adapter: "npm:^9.0.1" checksum: 10c0/8a4657aa6c0f0bc5d1fe77c2cd9603a3b07d4acefa634f1c5151190eed69711e7e599dd09c07915939a418dc8770d87e3529ecf1b029f2a9af7f2172d83acb1c languageName: node - linkType: soft + linkType: hard "locate-path@npm:^5.0.0": version: 5.0.0 @@ -9430,14 +9430,7 @@ __metadata: languageName: node linkType: hard -"loglevel@npm:1.9.1": - version: 1.9.1 - resolution: "loglevel@npm:1.9.1" - checksum: 10c0/152f0501cea367cf998c844a38b19f0b5af555756ad7d8650214a1f8c6a5b045e31b8cf5dae27d28339a061624ce3f618aadb333aed386cac041d6ddc5101a39 - languageName: node - linkType: hard - -"loglevel@npm:^1.9.1, loglevel@npm:^1.9.2": +"loglevel@npm:1.9.2": version: 1.9.2 resolution: "loglevel@npm:1.9.2" checksum: 10c0/1e317fa4648fe0b4a4cffef6de037340592cee8547b07d4ce97a487abe9153e704b98451100c799b032c72bb89c9366d71c9fb8192ada8703269263ae77acdc7 From 72c21877351a5ea8f50903affa1d6f74c804a4a3 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 11:24:39 +0200 Subject: [PATCH 06/14] fixup lint --- src/e2ee/matrixKeyProvider.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index d3fe4c640..350fb4be4 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -9,7 +9,7 @@ import { BaseKeyProvider, importKey, KeyProviderEvent, - RatchetResult + type RatchetResult } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { From cddd71f02650ce1d968469747a5810d84ade678d Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 11:28:19 +0200 Subject: [PATCH 07/14] fixup prettier --- src/e2ee/matrixKeyProvider.ts | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index 350fb4be4..fd6f17176 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -9,7 +9,7 @@ import { BaseKeyProvider, importKey, KeyProviderEvent, - type RatchetResult + type RatchetResult, } from "livekit-client"; import { logger } from "matrix-js-sdk/lib/logger"; import { @@ -20,14 +20,28 @@ import { export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; - private readonly onKeyRatchetComplete: (ratchetResult: RatchetResult, participantIdentity?: string, keyIndex?: number) => void; + private readonly onKeyRatchetComplete: ( + ratchetResult: RatchetResult, + participantIdentity?: string, + keyIndex?: number, + ) => void; public constructor() { super({ ratchetWindowSize: 10, keyringSize: 10 }); - this.onKeyRatchetComplete = (ratchetResult: RatchetResult, participantIdentity?: string, keyIndex?: number): void => { - logger.debug(`key ratcheted event received for ${participantIdentity} at index ${keyIndex}`); - this.rtcSession?.onKeyRatcheted(ratchetResult.chainKey,participantIdentity, keyIndex); + this.onKeyRatchetComplete = ( + ratchetResult: RatchetResult, + participantIdentity?: string, + keyIndex?: number, + ): void => { + logger.debug( + `key ratcheted event received for ${participantIdentity} at index ${keyIndex}`, + ); + this.rtcSession?.onKeyRatcheted( + ratchetResult.chainKey, + participantIdentity, + keyIndex, + ); }; this.on(KeyProviderEvent.KeyRatcheted, this.onKeyRatchetComplete); @@ -43,7 +57,6 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyQueryRatchetStep, this.doRatchetKey, ); - } this.rtcSession = rtcSession; @@ -58,22 +71,21 @@ export class MatrixKeyProvider extends BaseKeyProvider { this.doRatchetKey, ); - // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } - private doRatchetKey = (participantId:string, keyIndex:number): void => { - this.ratchetKey(participantId, keyIndex); - } + private doRatchetKey = (participantId: string, keyIndex: number): void => { + this.ratchetKey(participantId, keyIndex); + }; private onEncryptionKeyChanged = ( encryptionKey: Uint8Array, encryptionKeyIndex: number, participantId: string, ): void => { - importKey(encryptionKey, "HKDF", 'derive').then( + importKey(encryptionKey, "HKDF", "derive").then( (keyMaterial) => { this.onSetEncryptionKey(keyMaterial, participantId, encryptionKeyIndex); From 1d762cd2487c1f4913ba5a82b2cb018bc1718ffd Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 11:31:57 +0200 Subject: [PATCH 08/14] fixup tsconfig lib for PromiseWithResolvers --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index be12658ec..07632f206 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -3,7 +3,7 @@ "target": "es2022", "module": "es2022", "jsx": "react-jsx", - "lib": ["es2022", "dom", "dom.iterable"], + "lib": ["es2024", "dom", "dom.iterable"], // From Matrix-JS-SDK "strict": true, From 1095cb66e396bfbe8e71f6fc989265a5202c1f06 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 12:28:32 +0200 Subject: [PATCH 09/14] bump use key delay --- src/rtcSessionHelpers.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts index 51bc79b8d..875033356 100644 --- a/src/rtcSessionHelpers.ts +++ b/src/rtcSessionHelpers.ts @@ -126,6 +126,7 @@ export async function enterRTCSession( membershipKeepAlivePeriod: matrixRtcSessionConfig?.membership_keep_alive_period, makeKeyDelay: matrixRtcSessionConfig?.key_rotation_on_leave_delay, + useKeyDelay: 5_000, useExperimentalToDeviceTransport, }, ); From 08df63e68ae2e98bd8967a6437449b7588f3e1ae Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 14:39:04 +0200 Subject: [PATCH 10/14] tmp: Debug show device id in display name if devtool --- src/room/InCallView.tsx | 11 ++++++++++- src/state/CallViewModel.test.ts | 1 + src/state/CallViewModel.ts | 8 +++++++- src/utils/test-viewmodel.ts | 1 + 4 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b434a1da1..4b25a7362 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -96,6 +96,7 @@ import { ReactionsOverlay } from "./ReactionsOverlay"; import { CallEventAudioRenderer } from "./CallEventAudioRenderer"; import { debugTileLayout as debugTileLayoutSetting, + developerMode, useExperimentalToDeviceTransportSetting, useSetting, } from "../settings/settings"; @@ -134,6 +135,7 @@ export const ActiveCall: FC = (props) => { }; // eslint-disable-next-line react-hooks/exhaustive-deps }, []); + const [showDeveloperModeDebugOptions] = useSetting(developerMode); useEffect(() => { if (livekitRoom !== undefined) { @@ -145,6 +147,7 @@ export const ActiveCall: FC = (props) => { connStateObservable$, reactionsReader.raisedHands$, reactionsReader.reactions$, + showDeveloperModeDebugOptions, ); setVm(vm); return (): void => { @@ -152,7 +155,13 @@ export const ActiveCall: FC = (props) => { reactionsReader.destroy(); }; } - }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable$]); + }, [ + props.rtcSession, + livekitRoom, + props.e2eeSystem, + connStateObservable$, + showDeveloperModeDebugOptions, + ]); if (livekitRoom === undefined || vm === null) return null; diff --git a/src/state/CallViewModel.test.ts b/src/state/CallViewModel.test.ts index f0066476d..f60dec442 100644 --- a/src/state/CallViewModel.test.ts +++ b/src/state/CallViewModel.test.ts @@ -268,6 +268,7 @@ function withCallViewModel( connectionState$, raisedHands$, new BehaviorSubject({}), + false, ); onTestFinished(() => { diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 8fd6f819a..34397eac9 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -251,6 +251,7 @@ class UserMedia { public readonly speaker$: Observable; public readonly presenter$: Observable; + public constructor( public readonly id: string, member: RoomMember | undefined, @@ -487,7 +488,11 @@ export class CallViewModel extends ViewModel { const disambiguate = shouldDisambiguate(member, memberships, room); displaynameMap.set( matrixIdentifier, - calculateDisplayName(member, disambiguate), + this.showDeveloperModeDebugOptions + ? calculateDisplayName(member, disambiguate) + + " | " + + rtcMember.deviceId + : calculateDisplayName(member, disambiguate), ); } return displaynameMap; @@ -1344,6 +1349,7 @@ export class CallViewModel extends ViewModel { private readonly reactionsSubject$: Observable< Record >, + private readonly showDeveloperModeDebugOptions: boolean, ) { super(); } diff --git a/src/utils/test-viewmodel.ts b/src/utils/test-viewmodel.ts index c8a93c735..2c8be7aa6 100644 --- a/src/utils/test-viewmodel.ts +++ b/src/utils/test-viewmodel.ts @@ -138,6 +138,7 @@ export function getBasicCallViewModelEnvironment( of(ConnectionState.Connected), handRaisedSubject$, reactionsSubject$, + false, ); return { vm, From cc617c134722266734e1a5f17360c668743eb751 Mon Sep 17 00:00:00 2001 From: Valere Date: Tue, 13 May 2025 14:54:20 +0200 Subject: [PATCH 11/14] fixup 1095cb66e396bfbe8e71f6fc989265a5202c1f06 test --- src/rtcSessionHelpers.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/rtcSessionHelpers.test.ts b/src/rtcSessionHelpers.test.ts index ecfd44f74..e5763bb91 100644 --- a/src/rtcSessionHelpers.test.ts +++ b/src/rtcSessionHelpers.test.ts @@ -113,6 +113,7 @@ test("It joins the correct Session", async () => { useLegacyMemberEvents: false, useNewMembershipManager: true, useExperimentalToDeviceTransport: false, + useKeyDelay: 5000, }, ); }); From 85271f416b828b5121b7f93d9ace87658fb58ebd Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 14 May 2025 09:59:56 +0200 Subject: [PATCH 12/14] TO REVERT: More logs to debug --- src/e2ee/matrixKeyProvider.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index fd6f17176..c827e9875 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -16,6 +16,11 @@ import { type MatrixRTCSession, MatrixRTCSessionEvent, } from "matrix-js-sdk/lib/matrixrtc"; +import { encodeBase64 } from "matrix-js-sdk"; + +function tmpLogPrefix(msg: string): void { + logger.debug(`[RTCEncryption][MatrixKeyProvider] - ${msg}`); +} export class MatrixKeyProvider extends BaseKeyProvider { private rtcSession?: MatrixRTCSession; @@ -34,8 +39,9 @@ export class MatrixKeyProvider extends BaseKeyProvider { participantIdentity?: string, keyIndex?: number, ): void => { - logger.debug( + tmpLogPrefix( `key ratcheted event received for ${participantIdentity} at index ${keyIndex}`, + `key=${encodeBase64(new Uint8Array(ratchetResult.chainKey))}`, ); this.rtcSession?.onKeyRatcheted( ratchetResult.chainKey, @@ -70,13 +76,16 @@ export class MatrixKeyProvider extends BaseKeyProvider { MatrixRTCSessionEvent.EncryptionKeyQueryRatchetStep, this.doRatchetKey, ); - + tmpLogPrefix(`setRTCSession SESSION CHANGED`); // The new session could be aware of keys of which the old session wasn't, // so emit key changed events this.rtcSession.reemitEncryptionKeys(); } private doRatchetKey = (participantId: string, keyIndex: number): void => { + tmpLogPrefix( + `doRatchetKey participantId=${participantId} keyIndex=${keyIndex}`, + ); this.ratchetKey(participantId, keyIndex); }; @@ -85,11 +94,14 @@ export class MatrixKeyProvider extends BaseKeyProvider { encryptionKeyIndex: number, participantId: string, ): void => { + tmpLogPrefix( + `onEncryptionKeyChanged participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex} key=${encodeBase64(encryptionKey)}`, + ); importKey(encryptionKey, "HKDF", "derive").then( (keyMaterial) => { this.onSetEncryptionKey(keyMaterial, participantId, encryptionKeyIndex); - logger.debug( + tmpLogPrefix( `Sent new key to livekit room=${this.rtcSession?.room.roomId} participantId=${participantId} encryptionKeyIndex=${encryptionKeyIndex}`, ); }, From 4631e6e39d5dddc73a865309dcdf7e3c2b396c70 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 14 May 2025 10:16:26 +0200 Subject: [PATCH 13/14] fixup --- src/e2ee/matrixKeyProvider.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/e2ee/matrixKeyProvider.ts b/src/e2ee/matrixKeyProvider.ts index c827e9875..cbef32283 100644 --- a/src/e2ee/matrixKeyProvider.ts +++ b/src/e2ee/matrixKeyProvider.ts @@ -40,8 +40,7 @@ export class MatrixKeyProvider extends BaseKeyProvider { keyIndex?: number, ): void => { tmpLogPrefix( - `key ratcheted event received for ${participantIdentity} at index ${keyIndex}`, - `key=${encodeBase64(new Uint8Array(ratchetResult.chainKey))}`, + `key ratcheted event received for ${participantIdentity} at index ${keyIndex} key=${encodeBase64(new Uint8Array(ratchetResult.chainKey))}`, ); this.rtcSession?.onKeyRatcheted( ratchetResult.chainKey, From f20dd72062f3ff249a51fc7cc37d7e3c98f89df7 Mon Sep 17 00:00:00 2001 From: Valere Date: Wed, 14 May 2025 11:15:04 +0200 Subject: [PATCH 14/14] update js-sdk pin for more logs --- package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index dead409ca..1c17d42eb 100644 --- a/package.json +++ b/package.json @@ -102,7 +102,7 @@ "livekit-client": "^2.12.0", "lodash-es": "^4.17.21", "loglevel": "^1.9.1", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f8c27727e34642ef08936d801ce12fa4ef2940c2", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#ba2a87cc204c2b33abd63735936b281b07692815", "matrix-widget-api": "1.11.0", "normalize.css": "^8.0.1", "observable-hooks": "^4.2.3", diff --git a/yarn.lock b/yarn.lock index 1ebb6125d..1fac138d6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7000,7 +7000,7 @@ __metadata: livekit-client: "npm:^2.12.0" lodash-es: "npm:^4.17.21" loglevel: "npm:^1.9.1" - matrix-js-sdk: "github:matrix-org/matrix-js-sdk#f8c27727e34642ef08936d801ce12fa4ef2940c2" + matrix-js-sdk: "github:matrix-org/matrix-js-sdk#ba2a87cc204c2b33abd63735936b281b07692815" matrix-widget-api: "npm:1.11.0" normalize.css: "npm:^8.0.1" observable-hooks: "npm:^4.2.3" @@ -9584,9 +9584,9 @@ __metadata: languageName: node linkType: hard -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f8c27727e34642ef08936d801ce12fa4ef2940c2": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#ba2a87cc204c2b33abd63735936b281b07692815": version: 37.5.0 - resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f8c27727e34642ef08936d801ce12fa4ef2940c2" + resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=ba2a87cc204c2b33abd63735936b281b07692815" dependencies: "@babel/runtime": "npm:^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.0.1" @@ -9603,7 +9603,7 @@ __metadata: sdp-transform: "npm:^2.14.1" unhomoglyph: "npm:^1.0.6" uuid: "npm:11" - checksum: 10c0/d05a7491545ff9b68299c763573f378ec09e05437cfd23677a9c97409d4a862ca4d9d73a1adbc624b92a999ba0ddb6d0a105f24cfe9cd669fd42a942a95f25ca + checksum: 10c0/bef6ac481a56189aceedb9a784652a0562436cc24aea3e6986c8b47f583f38c8abbad22d55520a6339f690c2625dd12d41dd68533b4ed5bc4e340b8dc7476ab3 languageName: node linkType: hard