Add sticky event support (#3513)

* add sticky event support
 - use new js-sdk
 - use custom synapse
 - don't filter rooms by existing call state events

Signed-off-by: Timo K <toger5@hotmail.de>

* enable sticky events in the joinSessionConfig

Signed-off-by: Timo K <toger5@hotmail.de>

* Remove unused useNewMembershipmanager setting

* Add prefer sticky setting]

* Fixup call detection logic to allow sticky events

* lint

* update docker image

* More tidy

* update checksum

* bump js-sdk fix sticky events type

Signed-off-by: Timo K <toger5@hotmail.de>

* fix demo

Signed-off-by: Timo K <toger5@hotmail.de>

* always use multi sfu if we are using sticky events.

Signed-off-by: Timo K <toger5@hotmail.de>

* review

Signed-off-by: Timo K <toger5@hotmail.de>

* lint

Signed-off-by: Timo K <toger5@hotmail.de>

* Always consider multi-SFU mode enabled when using sticky events

CallViewModel would pass the wrong transport to enterRtcSession when the user enabled sticky events but didn't manually enable multi-SFU mode as well. This likely would've added some confusion to our attempts to test these modes.

* Fix test type errors

* add todo comment

Signed-off-by: Timo K <toger5@hotmail.de>

---------

Signed-off-by: Timo K <toger5@hotmail.de>
Co-authored-by: Half-Shot <will@half-shot.uk>
Co-authored-by: Robin <robin@robin.town>

backend/dev_homeserver.yaml

@@ -38,6 +38,8 @@ experimental_features:
   # MSC4222 needed for syncv2 state_after. This allow clients to
   # correctly track the state of the room.
   msc4222_enabled: true
+  # sticky events for matrixRTC user state
+  msc4354_enabled: true
 
 # The maximum allowed duration by which sent events can be delayed, as
 # per MSC4140. Must be a positive value if set.  Defaults to no

dev-backend-docker-compose.yml

@@ -88,7 +88,7 @@ services:
 
   synapse:
     hostname: homeserver
-    image: docker.io/matrixdotorg/synapse:latest
+    image: ghcr.io/element-hq/synapse:msc4354-5
     pull_policy: always
     environment:
       - SYNAPSE_CONFIG_PATH=/data/cfg/homeserver.yaml

locales/en/app.json

@@ -74,9 +74,12 @@
     "matrix_id": "Matrix ID: {{id}}",
     "multi_sfu": "Multi-SFU media transport",
     "mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
+    "prefer_sticky_events": {
+      "description": "Improves reliability of calls (requires homeserver support)",
+      "label": "Prefer sticky events"
+    },
     "show_connection_stats": "Show connection statistics",
     "url_params": "URL parameters",
-    "use_new_membership_manager": "Use the new implementation of the call MembershipManager",
     "use_to_device_key_transport": "Use to device key transport. This will fallback to room key transport when another call member sent a room key"
   },
   "disconnected_banner": "Connectivity to the server has been lost.",

package.json

@@ -109,7 +109,7 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=develop",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21",
     "matrix-widget-api": "^1.13.0",
     "normalize.css": "^8.0.1",
     "observable-hooks": "^4.2.3",

src/home/useGroupCallRooms.ts

@@ -20,7 +20,6 @@ import {
   MatrixRTCSessionManagerEvents,
   type MatrixRTCSession,
 } from "matrix-js-sdk/lib/matrixrtc";
-import { logger } from "matrix-js-sdk/lib/logger";
 
 import { getKeyForRoom } from "../e2ee/sharedKeyManagement";
 
@@ -114,19 +113,49 @@ const roomIsJoinable = (room: Room): boolean => {
   }
 };
 
+/**
+ * Determines if a given room has call events in it, and therefore
+ * is likely to be a call room.
+ * @param room The Matrix room instance.
+ * @returns `true` if the room has call events.
+ */
 const roomHasCallMembershipEvents = (room: Room): boolean => {
-  switch (room.getMyMembership()) {
+  // Check our room membership first, to rule out any rooms
-    case KnownMembership.Join:
+  // we can't have a call in.
-      return !!room
+  const myMembership = room.getMyMembership();
-        .getLiveTimeline()
+  if (myMembership === KnownMembership.Knock) {
-        .getState(EventTimeline.FORWARDS)
+    // Assume that a room you've knocked on is able to hold calls
-        ?.events?.get(EventType.GroupCallMemberPrefix);
+    return true;
-    case KnownMembership.Knock:
+  } else if (myMembership !== KnownMembership.Join) {
-      // Assume that a room you've knocked on is able to hold calls
+    // Otherwise, non-joined rooms should never show up.
-      return true;
+    return false;
-    default:
-      return false;
   }
+
+  // Legacy member state checks (cheaper to check.)
+  const timeline = room.getLiveTimeline();
+  if (
+    timeline
+      .getState(EventTimeline.FORWARDS)
+      ?.events?.has(EventType.GroupCallMemberPrefix)
+  ) {
+    return true;
+  }
+
+  // Check for *active* calls using sticky events.
+  for (const sticky of room._unstable_getStickyEvents()) {
+    if (sticky.getType() === EventType.RTCMembership) {
+      return true;
+    }
+  }
+
+  // Otherwise, check recent event history to see if anyone had
+  // sent a call membership in here.
+  return timeline.getEvents().some(
+    (e) =>
+      // Membership events only count if both of these are true
+      e.unstableStickyInfo && e.getType() === EventType.GroupCallMemberPrefix,
+  );
+  // Otherwise, it's *unlikely* this room was ever a call.
 };
 
 export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] {
@@ -140,24 +169,22 @@ export function useGroupCallRooms(client: MatrixClient): GroupCallRoom[] {
         .filter(roomHasCallMembershipEvents)
         .filter(roomIsJoinable);
       const sortedRooms = sortRooms(client, rooms);
-      Promise.all(
+      const items = sortedRooms.map((room) => {
-        sortedRooms.map((room) => {
+        const session = client.matrixRTC.getRoomSession(room);
-          const session = client.matrixRTC.getRoomSession(room);
+        return {
-          return {
+          roomAlias: room.getCanonicalAlias() ?? undefined,
-            roomAlias: room.getCanonicalAlias() ?? undefined,
+          roomName: room.name,
-            roomName: room.name,
+          avatarUrl: room.getMxcAvatarUrl()!,
-            avatarUrl: room.getMxcAvatarUrl()!,
+          room,
-            room,
+          session,
-            session,
+          participants: session.memberships
-            participants: session.memberships
+            .filter((m) => m.sender)
-              .filter((m) => m.userId)
+            .map((m) => room.getMember(m.sender!))
-              .map((m) => room.getMember(m.userId!))
+            .filter((m) => m) as RoomMember[],
-              .filter((m) => m) as RoomMember[],
+        };
-          };
+      });
-        }),
+
-      )
+      setRooms(items);
-        .then((items) => setRooms(items))
-        .catch(logger.error);
     }
 
     updateRooms();

src/room/GroupCallView.tsx

@@ -70,10 +70,6 @@ import {
   UnknownCallError,
 } from "../utils/errors.ts";
 import { GroupCallErrorBoundary } from "./GroupCallErrorBoundary.tsx";
-import {
-  useNewMembershipManager as useNewMembershipManagerSetting,
-  useSetting,
-} from "../settings/settings";
 import { useTypedEventEmitter } from "../useEvents";
 import { muteAllAudio$ } from "../state/MuteAllAudioModel.ts";
 import { useAppBarTitle } from "../AppBar.tsx";
@@ -186,7 +182,6 @@ export const GroupCallView: FC<Props> = ({
     password: passwordFromUrl,
   } = useUrlParams();
   const e2eeSystem = useRoomEncryptionSystem(room.roomId);
-  const [useNewMembershipManager] = useSetting(useNewMembershipManagerSetting);
 
   // Save the password once we start the groupCallView
   useEffect(() => {
@@ -310,7 +305,6 @@ export const GroupCallView: FC<Props> = ({
     mediaDevices,
     latestMuteStates,
     setJoined,
-    useNewMembershipManager,
   ]);
 
   // TODO refactor this + "joined" to just one callState

src/rtcSessionHelpers.test.ts

@@ -96,6 +96,7 @@ test("It joins the correct Session", async () => {
     {
       encryptMedia: true,
       useMultiSfu: USE_MUTI_SFU,
+      preferStickyEvents: false,
     },
   );
 
@@ -111,7 +112,6 @@ test("It joins the correct Session", async () => {
     expect.objectContaining({
       manageMediaKeys: true,
       useLegacyMemberEvents: false,
-      useNewMembershipManager: true,
       useExperimentalToDeviceTransport: false,
     }),
   );
@@ -197,6 +197,7 @@ test("It should not fail with configuration error if homeserver config has livek
     {
       encryptMedia: true,
       useMultiSfu: USE_MUTI_SFU,
+      preferStickyEvents: false,
     },
   );
 });

src/rtcSessionHelpers.ts

@@ -99,12 +99,11 @@ export async function makeTransport(
 
 export interface EnterRTCSessionOptions {
   encryptMedia: boolean;
-  // TODO: remove this flag, the new membership manager is stable enough
-  useNewMembershipManager?: boolean;
   // TODO: remove this flag, to-device transport is stable enough now
   useExperimentalToDeviceTransport?: boolean;
   /** EXPERIMENTAL: If true, will use the multi-sfu codepath where each member connects to its SFU instead of everyone connecting to an elected on. */
-  useMultiSfu?: boolean;
+  useMultiSfu: boolean;
+  preferStickyEvents: boolean;
 }
 
 /**
@@ -116,20 +115,13 @@ export interface EnterRTCSessionOptions {
 export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
   transport: LivekitTransport,
-  options: EnterRTCSessionOptions = {
+  {
-    encryptMedia: true,
-    useNewMembershipManager: true,
-    useExperimentalToDeviceTransport: false,
-    useMultiSfu: true,
-  },
-): Promise<void> {
-  const {
     encryptMedia,
-    useNewMembershipManager = true,
     useExperimentalToDeviceTransport = false,
-    useMultiSfu = true,
+    useMultiSfu,
-  } = options;
+    preferStickyEvents,
-
+  }: EnterRTCSessionOptions,
+): Promise<void> {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
 
@@ -148,7 +140,6 @@ export async function enterRTCSession(
     {
       notificationType,
       callIntent,
-      useNewMembershipManager,
       manageMediaKeys: encryptMedia,
       ...(useDeviceSessionMemberEvents !== undefined && {
         useLegacyMemberEvents: !useDeviceSessionMemberEvents,
@@ -164,6 +155,7 @@ export async function enterRTCSession(
       membershipEventExpiryMs:
         matrixRtcSessionConfig?.membership_event_expiry_ms,
       useExperimentalToDeviceTransport,
+      unstableSendStickyEvents: preferStickyEvents,
     },
   );
   if (widget) {

src/settings/DeveloperSettingsTab.tsx

@@ -5,8 +5,20 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { type ChangeEvent, type FC, useCallback, useMemo } from "react";
+import {
+  type ChangeEvent,
+  type FC,
+  useCallback,
+  useEffect,
+  useMemo,
+  useState,
+} from "react";
 import { useTranslation } from "react-i18next";
+import {
+  UNSTABLE_MSC4354_STICKY_EVENTS,
+  type MatrixClient,
+} from "matrix-js-sdk";
+import { logger } from "matrix-js-sdk/lib/logger";
 
 import { FieldRow, InputField } from "../input/Input";
 import {
@@ -14,16 +26,16 @@ import {
   duplicateTiles as duplicateTilesSetting,
   debugTileLayout as debugTileLayoutSetting,
   showConnectionStats as showConnectionStatsSetting,
-  useNewMembershipManager as useNewMembershipManagerSetting,
   useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
   multiSfu as multiSfuSetting,
   muteAllAudio as muteAllAudioSetting,
   alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
+  preferStickyEvents as preferStickyEventsSetting,
 } from "./settings";
-import type { MatrixClient } from "matrix-js-sdk";
 import type { Room as LivekitRoom } from "livekit-client";
 import styles from "./DeveloperSettingsTab.module.css";
 import { useUrlParams } from "../UrlParams";
+
 interface Props {
   client: MatrixClient;
   livekitRooms?: { room: LivekitRoom; url: string; isLocal?: boolean }[];
@@ -36,12 +48,24 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
     debugTileLayoutSetting,
   );
 
-  const [showConnectionStats, setShowConnectionStats] = useSetting(
+  const [stickyEventsSupported, setStickyEventsSupported] = useState(false);
-    showConnectionStatsSetting,
+  useEffect(() => {
+    client
+      .doesServerSupportUnstableFeature(UNSTABLE_MSC4354_STICKY_EVENTS)
+      .then((result) => {
+        setStickyEventsSupported(result);
+      })
+      .catch((ex) => {
+        logger.warn("Failed to check if sticky events are supported", ex);
+      });
+  }, [client]);
+
+  const [preferStickyEvents, setPreferStickyEvents] = useSetting(
+    preferStickyEventsSetting,
   );
 
-  const [useNewMembershipManager, setNewMembershipManager] = useSetting(
+  const [showConnectionStats, setShowConnectionStats] = useSetting(
-    useNewMembershipManagerSetting,
+    showConnectionStatsSetting,
   );
 
   const [alwaysShowIphoneEarpiece, setAlwaysShowIphoneEarpiece] = useSetting(
@@ -126,6 +150,22 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
           }
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="preferStickyEvents"
+          type="checkbox"
+          label={t("developer_mode.prefer_sticky_events.label")}
+          disabled={!stickyEventsSupported}
+          description={t("developer_mode.prefer_sticky_events.description")}
+          checked={!!preferStickyEvents}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setPreferStickyEvents(event.target.checked);
+            },
+            [setPreferStickyEvents],
+          )}
+        />
+      </FieldRow>
       <FieldRow>
         <InputField
           id="showConnectionStats"
@@ -140,20 +180,6 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
           )}
         />
       </FieldRow>
-      <FieldRow>
-        <InputField
-          id="useNewMembershipManager"
-          type="checkbox"
-          label={t("developer_mode.use_new_membership_manager")}
-          checked={!!useNewMembershipManager}
-          onChange={useCallback(
-            (event: ChangeEvent<HTMLInputElement>): void => {
-              setNewMembershipManager(event.target.checked);
-            },
-            [setNewMembershipManager],
-          )}
-        />
-      </FieldRow>
       <FieldRow>
         <InputField
           id="useToDeviceKeyTransport"
@@ -173,7 +199,9 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
           id="multiSfu"
           type="checkbox"
           label={t("developer_mode.multi_sfu")}
-          checked={multiSfu}
+          // If using sticky events we implicitly prefer use multi-sfu
+          checked={multiSfu || preferStickyEvents}
+          disabled={preferStickyEvents}
           onChange={useCallback(
             (event: ChangeEvent<HTMLInputElement>): void => {
               setMultiSfu(event.target.checked);

src/settings/settings.ts

@@ -83,6 +83,11 @@ export const showConnectionStats = new Setting<boolean>(
   false,
 );
 
+export const preferStickyEvents = new Setting<boolean>(
+  "prefer-sticky-events",
+  false,
+);
+
 export const audioInput = new Setting<string | undefined>(
   "audio-input",
   undefined,
@@ -115,11 +120,6 @@ export const soundEffectVolume = new Setting<number>(
   0.5,
 );
 
-export const useNewMembershipManager = new Setting<boolean>(
-  "new-membership-manager",
-  true,
-);
-
 export const useExperimentalToDeviceTransport = new Setting<boolean>(
   "experimental-to-device-transport",
   true,

src/state/CallViewModel.ts

@@ -90,6 +90,7 @@ import {
   duplicateTiles,
   multiSfu,
   playReactionsSound,
+  preferStickyEvents,
   showReactions,
 } from "../settings/settings";
 import { isFirefox } from "../Platform";
@@ -265,23 +266,33 @@ export class CallViewModel {
   /**
    * Lists the transports used by ourselves, plus all other MatrixRTC session
    * members. For completeness this also lists the preferred transport and
-   * whether we are in multi-SFU mode (because advertisedTransport$ wants to
+   * whether we are in multi-SFU mode or sticky events mode (because
-   * read them at the same time, and bundling data together when it might change
+   * advertisedTransport$ wants to read them at the same time, and bundling data
-   * together is what you have to do in RxJS to avoid reading inconsistent state
+   * together when it might change together is what you have to do in RxJS to
-   * or observing too many changes.)
+   * avoid reading inconsistent state or observing too many changes.)
    */
+  // TODO-MULTI-SFU find a better name for this. with the addition of sticky events it's no longer just about transports.
   private readonly transports$: Behavior<{
     local: Async<LivekitTransport>;
     remote: { membership: CallMembership; transport: LivekitTransport }[];
     preferred: Async<LivekitTransport>;
     multiSfu: boolean;
+    preferStickyEvents: boolean;
   } | null> = this.scope.behavior(
     this.joined$.pipe(
       switchMap((joined) =>
         joined
           ? combineLatest(
-              [this.preferredTransport$, this.memberships$, multiSfu.value$],
+              [
-              (preferred, memberships, multiSfu) => {
+                this.preferredTransport$,
+                this.memberships$,
+                multiSfu.value$,
+                preferStickyEvents.value$,
+              ],
+              (preferred, memberships, preferMultiSfu, preferStickyEvents) => {
+                // Multi-SFU must be implicitly enabled when using sticky events
+                const multiSfu = preferStickyEvents || preferMultiSfu;
+
                 const oldestMembership =
                   this.matrixRTCSession.getOldestMembership();
                 const remote = memberships.flatMap((m) => {
@@ -292,6 +303,7 @@ export class CallViewModel {
                     ? [{ membership: m, transport: t }]
                     : [];
                 });
+
                 let local = preferred;
                 if (!multiSfu) {
                   const oldest = this.matrixRTCSession.getOldestMembership();
@@ -302,6 +314,7 @@ export class CallViewModel {
                       local = ready(selection);
                   }
                 }
+
                 if (local.state === "error") {
                   this._configError$.next(
                     local.value instanceof ElementCallError
@@ -309,7 +322,14 @@ export class CallViewModel {
                       : new UnknownCallError(local.value),
                   );
                 }
-                return { local, remote, preferred, multiSfu };
+
+                return {
+                  local,
+                  remote,
+                  preferred,
+                  multiSfu,
+                  preferStickyEvents,
+                };
               },
             )
           : of(null),
@@ -339,10 +359,11 @@ export class CallViewModel {
 
   /**
    * The transport we should advertise in our MatrixRTC membership (plus whether
-   * it is a multi-SFU transport).
+   * it is a multi-SFU transport and whether we should use sticky events).
    */
   private readonly advertisedTransport$: Behavior<{
     multiSfu: boolean;
+    preferStickyEvents: boolean;
     transport: LivekitTransport;
   } | null> = this.scope.behavior(
     this.transports$.pipe(
@@ -351,6 +372,7 @@ export class CallViewModel {
         transports.preferred.state === "ready"
           ? {
               multiSfu: transports.multiSfu,
+              preferStickyEvents: transports.preferStickyEvents,
               // In non-multi-SFU mode we should always advertise the preferred
               // SFU to minimize the number of membership updates
               transport: transports.multiSfu
@@ -361,6 +383,7 @@ export class CallViewModel {
       ),
       distinctUntilChanged<{
         multiSfu: boolean;
+        preferStickyEvents: boolean;
         transport: LivekitTransport;
       } | null>(deepCompare),
     ),
@@ -1796,8 +1819,8 @@ export class CallViewModel {
           await enterRTCSession(this.matrixRTCSession, advertised.transport, {
             encryptMedia: this.options.encryptionSystem.kind !== E2eeType.NONE,
             useExperimentalToDeviceTransport: true,
-            useNewMembershipManager: true,
             useMultiSfu: advertised.multiSfu,
+            preferStickyEvents: advertised.preferStickyEvents,
           });
         } catch (e) {
           logger.error("Error entering RTC session", e);

yarn.lock

@@ -7545,7 +7545,7 @@ __metadata:
     livekit-client: "npm:^2.13.0"
     lodash-es: "npm:^4.17.21"
     loglevel: "npm:^1.9.1"
-    matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=develop"
+    matrix-js-sdk: "github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21"
     matrix-widget-api: "npm:^1.13.0"
     normalize.css: "npm:^8.0.1"
     observable-hooks: "npm:^4.2.3"
@@ -10335,9 +10335,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=develop":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#head=toger5/sticky-events&commit=e7f5bec51b6f70501a025b79fe5021c933385b21":
   version: 38.4.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=fd949fe486038099ee111a72b57ce711e85bc352"
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=e7f5bec51b6f70501a025b79fe5021c933385b21"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^15.3.0"
@@ -10353,7 +10353,7 @@ __metadata:
     sdp-transform: "npm:^2.14.1"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:13"
-  checksum: 10c0/d334811074726482b58089fef6c9e98a462fbc1d91c63798307648bdc1349d2e154aa31f391690e2c9cd90eee61a3be9fe8872873e7f828b529d9268d2a25b78
+  checksum: 10c0/7adffdc183affd2d3ee1e8497cad6ca7904a37f98328ff7bc15aa6c1829dc9f9a92f8e1bd6260432a33626ff2a839644de938270163e73438b7294675cd954e4
   languageName: node
   linkType: hard