Add option to enable to-device-encryption (#3167)

* enable to-device-encryption

* add logging for key provider

* make rooms encrypted

* add dev setting to choose to-device or room encryption

* add indicator when to-device is used.

locales/en/app.json

@@ -73,7 +73,8 @@
     "show_connection_stats": "Show connection statistics",
     "show_non_member_tiles": "Show tiles for non-member media",
     "url_params": "URL parameters",
-    "use_new_membership_manager": "Use the new implementation of the call MembershipManager"
+    "use_new_membership_manager": "Use the new implementation of the call MembershipManager",
+    "use_to_device_key_transport": "Use to device messages to distribute keys for matrixRTC media"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",
   "error": {

package.json

@@ -100,7 +100,7 @@
     "livekit-client": "2.11.1",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#8395919f0fd1af7cab1e793d736f2cdf18ef7686",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#e3a3a52f2a56cb5cc52b57b36e9a915faed0b5db",
     "matrix-widget-api": "1.11.0",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",

src/livekit/useLiveKit.ts

@@ -49,11 +49,14 @@ export function useLiveKit(
     if (e2eeSystem.kind === E2eeType.NONE) return undefined;
 
     if (e2eeSystem.kind === E2eeType.PER_PARTICIPANT) {
+      logger.info("Created MatrixKeyProvider (per participant)");
       return {
         keyProvider: new MatrixKeyProvider(),
         worker: new E2EEWorker(),
       };
     } else if (e2eeSystem.kind === E2eeType.SHARED_KEY && e2eeSystem.secret) {
+      logger.info("Created ExternalE2EEKeyProvider (shared key)");
+
       return {
         keyProvider: new ExternalE2EEKeyProvider(),
         worker: new E2EEWorker(),

src/room/GroupCallView.tsx

@@ -62,6 +62,7 @@ import {
 } from "../utils/errors.ts";
 import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx";
 import {
+  useExperimentalToDeviceTransportSetting,
   useNewMembershipManagerSetting as useNewMembershipManagerSetting,
   useSetting,
 } from "../settings/settings";
@@ -151,6 +152,9 @@ export const GroupCallView: FC<Props> = ({
   const { perParticipantE2EE, returnToLobby } = useUrlParams();
   const e2eeSystem = useRoomEncryptionSystem(room.roomId);
   const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting);
+  const [useExperimentalToDeviceTransport] = useSetting(
+    useExperimentalToDeviceTransportSetting,
+  );
 
   usePageTitle(roomName);
 
@@ -178,16 +182,13 @@ export const GroupCallView: FC<Props> = ({
   const latestMuteStates = useLatest(muteStates);
 
   const enterRTCSessionOrError = useCallback(
-    async (
+    async (rtcSession: MatrixRTCSession): Promise<void> => {
-      rtcSession: MatrixRTCSession,
-      perParticipantE2EE: boolean,
-      newMembershipManager: boolean,
-    ): Promise<void> => {
       try {
         await enterRTCSession(
           rtcSession,
           perParticipantE2EE,
-          newMembershipManager,
+          useNewMembershipManager,
+          useExperimentalToDeviceTransport,
         );
       } catch (e) {
         if (e instanceof ElementCallError) {
@@ -201,7 +202,11 @@ export const GroupCallView: FC<Props> = ({
         }
       }
     },
-    [setExternalError],
+    [
+      perParticipantE2EE,
+      useExperimentalToDeviceTransport,
+      useNewMembershipManager,
+    ],
   );
 
   useEffect(() => {
@@ -253,11 +258,7 @@ export const GroupCallView: FC<Props> = ({
               await defaultDeviceSetup(
                 ev.detail.data as unknown as JoinCallData,
               );
-              await enterRTCSessionOrError(
+              await enterRTCSessionOrError(rtcSession);
-                rtcSession,
-                perParticipantE2EE,
-                useNewMembershipManager,
-              );
               widget.api.transport.reply(ev.detail, {});
             })().catch((e) => {
               logger.error("Error joining RTC session", e);
@@ -270,21 +271,13 @@ export const GroupCallView: FC<Props> = ({
         } else {
           // No lobby and no preload: we enter the rtc session right away
           (async (): Promise<void> => {
-            await enterRTCSessionOrError(
+            await enterRTCSessionOrError(rtcSession);
-              rtcSession,
-              perParticipantE2EE,
-              useNewMembershipManager,
-            );
           })().catch((e) => {
             logger.error("Error joining RTC session", e);
           });
         }
       } else {
-        void enterRTCSessionOrError(
+        void enterRTCSessionOrError(rtcSession);
-          rtcSession,
-          perParticipantE2EE,
-          useNewMembershipManager,
-        );
       }
     }
   }, [
@@ -407,13 +400,7 @@ export const GroupCallView: FC<Props> = ({
         client={client}
         matrixInfo={matrixInfo}
         muteStates={muteStates}
-        onEnter={() =>
+        onEnter={() => void enterRTCSessionOrError(rtcSession)}
-          void enterRTCSessionOrError(
-            rtcSession,
-            perParticipantE2EE,
-            useNewMembershipManager,
-          )
-        }
         confineToRoom={confineToRoom}
         hideHeader={hideHeader}
         participantCount={participantCount}
@@ -491,11 +478,7 @@ export const GroupCallView: FC<Props> = ({
       recoveryActionHandler={(action) => {
         if (action == "reconnect") {
           setLeft(false);
-          enterRTCSessionOrError(
+          enterRTCSessionOrError(rtcSession).catch((e) => {
-            rtcSession,
-            perParticipantE2EE,
-            useNewMembershipManager,
-          ).catch((e) => {
             logger.error("Error re-entering RTC session", e);
           });
         }

src/room/InCallView.tsx

@@ -10,6 +10,7 @@ import {
   RoomContext,
   useLocalParticipant,
 } from "@livekit/components-react";
+import { Text } from "@vector-im/compound-web";
 import { ConnectionState, type Room } from "livekit-client";
 import { type MatrixClient } from "matrix-js-sdk";
 import {
@@ -94,11 +95,11 @@ import { ReactionsOverlay } from "./ReactionsOverlay";
 import { CallEventAudioRenderer } from "./CallEventAudioRenderer";
 import {
   debugTileLayout as debugTileLayoutSetting,
+  useExperimentalToDeviceTransportSetting,
   useSetting,
 } from "../settings/settings";
 import { ReactionsReader } from "../reactions/ReactionsReader";
 import { ConnectionLostError } from "../utils/errors.ts";
-
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
 const maxTapDurationMs = 400;
@@ -216,6 +217,10 @@ export const InCallView: FC<InCallViewProps> = ({
     room: livekitRoom,
   });
 
+  const [toDeviceEncryption] = useSetting(
+    useExperimentalToDeviceTransportSetting,
+  );
+
   const toggleMicrophone = useCallback(
     () => muteStates.audio.setEnabled?.((e) => !e),
     [muteStates],
@@ -662,6 +667,18 @@ export const InCallView: FC<InCallViewProps> = ({
             </RightNav>
           </Header>
         ))}
+      {
+        // TODO: remove this once we remove the developer flag
+        // and find a better way to device what key transport to use.
+        toDeviceEncryption && (
+          <Text
+            style={{ height: 0, zIndex: 1, alignSelf: "center", margin: 0 }}
+            size="sm"
+          >
+            using to Device key transport
+          </Text>
+        )
+      }
       <RoomAudioRenderer />
       {renderContent()}
       <CallEventAudioRenderer vm={vm} />

src/rtcSessionHelpers.test.ts

@@ -112,6 +112,7 @@ test("It joins the correct Session", async () => {
       manageMediaKeys: false,
       useLegacyMemberEvents: false,
       useNewMembershipManager: true,
+      useExperimentalToDeviceTransport: false,
     },
   );
 });

src/rtcSessionHelpers.ts

@@ -98,6 +98,7 @@ export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
   encryptMedia: boolean,
   useNewMembershipManager = true,
+  useExperimentalToDeviceTransport = false,
 ): Promise<void> {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
@@ -125,6 +126,7 @@ export async function enterRTCSession(
       membershipKeepAlivePeriod:
         matrixRtcSessionConfig?.membership_keep_alive_period,
       makeKeyDelay: matrixRtcSessionConfig?.key_rotation_on_leave_delay,
+      useExperimentalToDeviceTransport,
     },
   );
   if (widget) {

src/settings/DeveloperSettingsTab.tsx

@@ -16,6 +16,7 @@ import {
   showNonMemberTiles as showNonMemberTilesSetting,
   showConnectionStats as showConnectionStatsSetting,
   useNewMembershipManagerSetting,
+  useExperimentalToDeviceTransportSetting,
 } from "./settings";
 import type { MatrixClient } from "matrix-js-sdk";
 import type { Room as LivekitRoom } from "livekit-client";
@@ -44,6 +45,10 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
     useNewMembershipManagerSetting,
   );
 
+  const [
+    useExperimentalToDeviceTransport,
+    setUseExperimentalToDeviceTransport,
+  ] = useSetting(useExperimentalToDeviceTransportSetting);
   const urlParams = useUrlParams();
 
   const sfuUrl = useMemo((): URL | null => {
@@ -156,6 +161,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRoom }) => {
           )}
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="useToDeviceKeyTransport"
+          type="checkbox"
+          label={t("developer_mode.use_to_device_key_transport")}
+          checked={!!useExperimentalToDeviceTransport}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setUseExperimentalToDeviceTransport(event.target.checked);
+            },
+            [setUseExperimentalToDeviceTransport],
+          )}
+        />
+      </FieldRow>
       {livekitRoom ? (
         <>
           <p>

src/settings/settings.ts

@@ -117,4 +117,10 @@ export const useNewMembershipManagerSetting = new Setting<boolean>(
   "new-membership-manager",
   true,
 );
+
+export const useExperimentalToDeviceTransportSetting = new Setting<boolean>(
+  "experimental-to-device-transport",
+  false,
+);
+
 export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);

yarn.lock

@@ -6913,7 +6913,7 @@ __metadata:
     livekit-client: "npm:2.11.1"
     lodash-es: "npm:^4.17.21"
     loglevel: "npm:^1.9.1"
-    matrix-js-sdk: "github:matrix-org/matrix-js-sdk#8395919f0fd1af7cab1e793d736f2cdf18ef7686"
+    matrix-js-sdk: "github:matrix-org/matrix-js-sdk#e3a3a52f2a56cb5cc52b57b36e9a915faed0b5db"
     matrix-widget-api: "npm:1.11.0"
     normalize.css: "npm:^8.0.1"
     observable-hooks: "npm:^4.2.3"
@@ -9504,9 +9504,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#8395919f0fd1af7cab1e793d736f2cdf18ef7686":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#e3a3a52f2a56cb5cc52b57b36e9a915faed0b5db":
-  version: 37.1.0
+  version: 37.3.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=8395919f0fd1af7cab1e793d736f2cdf18ef7686"
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=e3a3a52f2a56cb5cc52b57b36e9a915faed0b5db"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^14.0.1"
@@ -9523,7 +9523,7 @@ __metadata:
     sdp-transform: "npm:^2.14.1"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:11"
-  checksum: 10c0/a0eb3be822e07cfe53965f6ca4f0c3cdf8ba3728d03a15f2322a463a7543206583e0c2f34d6b6d45089ce36eec60d77d9e90eb0635d3c65a343f77728908fe57
+  checksum: 10c0/1baf50f93576a6fdf46d76c7a84cf43adeb0b04e692165f749f15c56e8e3fd0f5f354a1702b9f9de1688cebbdee176f7056b71e8a526ef9b0fbbe23405c2aee2
   languageName: node
   linkType: hard