Turn multi-SFU media transport into a developer option

locales/en/app.json

@@ -72,6 +72,7 @@
     "livekit_server_info": "LiveKit Server Info",
     "livekit_sfu": "LiveKit SFU: {{url}}",
     "matrix_id": "Matrix ID: {{id}}",
+    "multi_sfu": "Multi-SFU media transport",
     "mute_all_audio": "Mute all audio (participants, reactions, join sounds)",
     "show_connection_stats": "Show connection statistics",
     "url_params": "URL parameters",
@@ -91,7 +92,7 @@
     "generic_description": "Submitting debug logs will help us track down the problem.",
     "insufficient_capacity": "Insufficient capacity",
     "insufficient_capacity_description": "The server has reached its maximum capacity and you cannot join the call at this time. Try again later, or contact your server admin if the problem persists.",
-    "matrix_rtc_focus_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
+    "matrix_rtc_transport_missing": "The server is not configured to work with {{brand}}. Please contact your server admin (Domain: {{domain}}, Error Code: {{ errorCode }}).",
     "open_elsewhere": "Opened in another tab",
     "open_elsewhere_description": "{{brand}} has been opened in another tab. If that doesn't sound right, try reloading the page.",
     "room_creation_restricted": "Failed to create call",

src/room/GroupCallErrorBoundary.test.tsx

@@ -26,7 +26,7 @@ import {
   E2EENotSupportedError,
   type ElementCallError,
   InsufficientCapacityError,
-  MatrixRTCFocusMissingError,
+  MatrixRTCTransportMissingError,
   UnknownCallError,
 } from "../utils/errors.ts";
 import { mockConfig } from "../utils/test.ts";
@@ -34,7 +34,7 @@ import { ElementWidgetActions, type WidgetHelpers } from "../widget.ts";
 
 test.each([
   {
-    error: new MatrixRTCFocusMissingError("example.com"),
+    error: new MatrixRTCTransportMissingError("example.com"),
     expectedTitle: "Call is not supported",
   },
   {
@@ -85,7 +85,7 @@ test.each([
 );
 
 test("should render the error page with link back to home", async () => {
-  const error = new MatrixRTCFocusMissingError("example.com");
+  const error = new MatrixRTCTransportMissingError("example.com");
   const TestComponent = (): ReactNode => {
     throw error;
   };
@@ -213,7 +213,7 @@ describe("Rageshake button", () => {
 });
 
 test("should have a close button in widget mode", async () => {
-  const error = new MatrixRTCFocusMissingError("example.com");
+  const error = new MatrixRTCTransportMissingError("example.com");
   const TestComponent = (): ReactNode => {
     throw error;
   };

src/room/GroupCallView.test.tsx

@@ -42,7 +42,7 @@ import {
 import { GroupCallView } from "./GroupCallView";
 import { type WidgetHelpers } from "../widget";
 import { LazyEventEmitter } from "../LazyEventEmitter";
-import { MatrixRTCFocusMissingError } from "../utils/errors";
+import { MatrixRTCTransportMissingError } from "../utils/errors";
 import { ProcessorProvider } from "../livekit/TrackProcessorContext";
 import { MediaDevicesContext } from "../MediaDevicesContext";
 import { HeaderStyle } from "../UrlParams";
@@ -258,7 +258,7 @@ test("GroupCallView leaves the session when an error occurs", async () => {
 
 test("GroupCallView shows errors that occur during joining", async () => {
   const user = userEvent.setup();
-  enterRTCSession.mockRejectedValue(new MatrixRTCFocusMissingError(""));
+  enterRTCSession.mockRejectedValue(new MatrixRTCTransportMissingError(""));
   onTestFinished(() => {
     enterRTCSession.mockReset();
   });

src/rtcSessionHelpers.ts

@@ -17,7 +17,7 @@ import { AutoDiscovery } from "matrix-js-sdk/lib/autodiscovery";
 import { PosthogAnalytics } from "./analytics/PosthogAnalytics";
 import { Config } from "./config/Config";
 import { ElementWidgetActions, widget, type WidgetHelpers } from "./widget";
-import { MatrixRTCFocusMissingError } from "./utils/errors";
+import { MatrixRTCTransportMissingError } from "./utils/errors";
 import { getUrlParams } from "./UrlParams";
 import { getSFUConfigWithOpenID } from "./livekit/openIDSFU.ts";
 
@@ -28,35 +28,31 @@ export function getLivekitAlias(rtcSession: MatrixRTCSession): string {
   return rtcSession.room.roomId;
 }
 
-async function makeFocusInternal(
+async function makeTransportInternal(
   rtcSession: MatrixRTCSession,
 ): Promise<LivekitTransport> {
-  logger.log("Searching for a preferred focus");
+  logger.log("Searching for a preferred transport");
   const livekitAlias = getLivekitAlias(rtcSession);
 
-  const urlFromStorage = localStorage.getItem("robin-matrixrtc-auth");
+  // TODO-MULTI-SFU: Either remove this dev tool or make it more official
+  const urlFromStorage =
+    localStorage.getItem("robin-matrixrtc-auth") ??
+    localStorage.getItem("timo-focus-url");
   if (urlFromStorage !== null) {
-    const focusFromStorage: LivekitTransport = {
+    const transportFromStorage: LivekitTransport = {
       type: "livekit",
       livekit_service_url: urlFromStorage,
       livekit_alias: livekitAlias,
     };
-    logger.log("Using LiveKit focus from local storage: ", focusFromStorage);
+    logger.log(
-    return focusFromStorage;
+      "Using LiveKit transport from local storage: ",
+      transportFromStorage,
+    );
+    return transportFromStorage;
   }
 
   // Prioritize the .well-known/matrix/client, if available, over the configured SFU
   const domain = rtcSession.room.client.getDomain();
-  if (localStorage.getItem("timo-focus-url")) {
-    const timoFocusUrl = localStorage.getItem("timo-focus-url")!;
-    const focusFromUrl: LivekitTransport = {
-      type: "livekit",
-      livekit_service_url: timoFocusUrl,
-      livekit_alias: livekitAlias,
-    };
-    logger.log("Using LiveKit focus from localStorage: ", timoFocusUrl);
-    return focusFromUrl;
-  }
   if (domain) {
     // we use AutoDiscovery instead of relying on the MatrixClient having already
     // been fully configured and started
@@ -64,46 +60,46 @@ async function makeFocusInternal(
       FOCI_WK_KEY
     ];
     if (Array.isArray(wellKnownFoci)) {
-      const focus: LivekitTransportConfig | undefined = wellKnownFoci.find(
+      const transport: LivekitTransportConfig | undefined = wellKnownFoci.find(
         (f) => f && isLivekitTransportConfig(f),
       );
-      if (focus !== undefined) {
+      if (transport !== undefined) {
-        logger.log("Using LiveKit focus from .well-known: ", focus);
+        logger.log("Using LiveKit transport from .well-known: ", transport);
-        return { ...focus, livekit_alias: livekitAlias };
+        return { ...transport, livekit_alias: livekitAlias };
       }
     }
   }
 
   const urlFromConf = Config.get().livekit?.livekit_service_url;
   if (urlFromConf) {
-    const focusFromConf: LivekitTransport = {
+    const transportFromConf: LivekitTransport = {
       type: "livekit",
       livekit_service_url: urlFromConf,
       livekit_alias: livekitAlias,
     };
-    logger.log("Using LiveKit focus from config: ", focusFromConf);
+    logger.log("Using LiveKit transport from config: ", transportFromConf);
-    return focusFromConf;
+    return transportFromConf;
   }
 
-  throw new MatrixRTCFocusMissingError(domain ?? "");
+  throw new MatrixRTCTransportMissingError(domain ?? "");
 }
 
-export async function makeFocus(
+export async function makeTransport(
   rtcSession: MatrixRTCSession,
 ): Promise<LivekitTransport> {
-  const focus = await makeFocusInternal(rtcSession);
+  const transport = await makeTransportInternal(rtcSession);
   // this will call the jwt/sfu/get endpoint to pre create the livekit room.
   await getSFUConfigWithOpenID(
     rtcSession.room.client,
-    focus.livekit_service_url,
+    transport.livekit_service_url,
-    focus.livekit_alias,
+    transport.livekit_alias,
   );
-  return focus;
+  return transport;
 }
 
 export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
-  focus: LivekitTransport,
+  transport: LivekitTransport,
   encryptMedia: boolean,
   useNewMembershipManager = true,
   useExperimentalToDeviceTransport = false,
@@ -120,10 +116,10 @@ export async function enterRTCSession(
   const useDeviceSessionMemberEvents =
     features?.feature_use_device_session_member_events;
   const { sendNotificationType: notificationType, callIntent } = getUrlParams();
-  // Multi-sfu does not need a focus preferred list. just the focus that is actually used.
+  // Multi-sfu does not need a preferred foci list. just the focus that is actually used.
   rtcSession.joinRoomSession(
-    useMultiSfu ? [focus] : [],
+    useMultiSfu ? [] : [transport],
-    useMultiSfu ? focus : undefined,
+    useMultiSfu ? transport : undefined,
     {
       notificationType,
       callIntent,

src/settings/DeveloperSettingsTab.tsx

@@ -16,6 +16,7 @@ import {
   showConnectionStats as showConnectionStatsSetting,
   useNewMembershipManager as useNewMembershipManagerSetting,
   useExperimentalToDeviceTransport as useExperimentalToDeviceTransportSetting,
+  multiSfu as multiSfuSetting,
   muteAllAudio as muteAllAudioSetting,
   alwaysShowIphoneEarpiece as alwaysShowIphoneEarpieceSetting,
 } from "./settings";
@@ -50,6 +51,7 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
     useExperimentalToDeviceTransport,
     setUseExperimentalToDeviceTransport,
   ] = useSetting(useExperimentalToDeviceTransportSetting);
+  const [multiSfu, setMultiSfu] = useSetting(multiSfuSetting);
 
   const [muteAllAudio, setMuteAllAudio] = useSetting(muteAllAudioSetting);
 
@@ -166,6 +168,20 @@ export const DeveloperSettingsTab: FC<Props> = ({ client, livekitRooms }) => {
           )}
         />
       </FieldRow>
+      <FieldRow>
+        <InputField
+          id="multiSfu"
+          type="checkbox"
+          label={t("developer_mode.multi_sfu")}
+          checked={multiSfu}
+          onChange={useCallback(
+            (event: ChangeEvent<HTMLInputElement>): void => {
+              setMultiSfu(event.target.checked);
+            },
+            [setMultiSfu],
+          )}
+        />
+      </FieldRow>
       <FieldRow>
         <InputField
           id="muteAllAudio"

src/settings/settings.ts

@@ -125,6 +125,8 @@ export const useExperimentalToDeviceTransport = new Setting<boolean>(
   true,
 );
 
+export const multiSfu = new Setting<boolean>("multi-sfu", false);
+
 export const muteAllAudio = new Setting<boolean>("mute-all-audio", false);
 
 export const alwaysShowSelf = new Setting<boolean>("always-show-self", true);

src/state/Async.ts

@@ -0,0 +1,44 @@
+/*
+Copyright 2025 New Vector Ltd.
+
+SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
+Please see LICENSE in the repository root for full details.
+*/
+
+import {
+  catchError,
+  from,
+  map,
+  Observable,
+  of,
+  startWith,
+  switchMap,
+} from "rxjs";
+
+export type Async<A> =
+  | { state: "loading" }
+  | { state: "error"; value: Error }
+  | { state: "ready"; value: A };
+
+export const loading: Async<never> = { state: "loading" };
+export function error(value: Error): Async<never> {
+  return { state: "error", value };
+}
+export function ready<A>(value: A): Async<A> {
+  return { state: "ready", value };
+}
+
+export function async<A>(promise: Promise<A>): Observable<Async<A>> {
+  return from(promise).pipe(
+    map(ready),
+    startWith(loading),
+    catchError((e) => of(error(e))),
+  );
+}
+
+export function mapAsync<A, B>(
+  async: Async<A>,
+  project: (value: A) => B,
+): Async<B> {
+  return async.state === "ready" ? ready(project(async.value)) : async;
+}

src/state/CallViewModel.ts

@@ -28,6 +28,7 @@ import {
   EventType,
   RoomEvent,
 } from "matrix-js-sdk";
+import { deepCompare } from "matrix-js-sdk/lib/utils";
 import {
   BehaviorSubject,
   EMPTY,
@@ -48,6 +49,7 @@ import {
   of,
   pairwise,
   race,
+  repeat,
   scan,
   skip,
   skipWhile,
@@ -57,6 +59,7 @@ import {
   switchScan,
   take,
   takeUntil,
+  takeWhile,
   tap,
   throttleTime,
   timer,
@@ -65,6 +68,7 @@ import { logger } from "matrix-js-sdk/lib/logger";
 import {
   type CallMembership,
   isLivekitTransport,
+  type LivekitTransport,
   type MatrixRTCSession,
   MatrixRTCSessionEvent,
   type MatrixRTCSessionEventHandlerMap,
@@ -90,6 +94,7 @@ import {
 import { ObservableScope } from "./ObservableScope";
 import {
   duplicateTiles,
+  multiSfu,
   playReactionsSound,
   showReactions,
 } from "../settings/settings";
@@ -118,7 +123,7 @@ import { constant, type Behavior } from "./Behavior";
 import {
   enterRTCSession,
   getLivekitAlias,
-  makeFocus,
+  makeTransport,
 } from "../rtcSessionHelpers";
 import { E2eeType } from "../e2ee/e2eeType";
 import { MatrixKeyProvider } from "../e2ee/matrixKeyProvider";
@@ -127,6 +132,7 @@ import { type MuteStates } from "./MuteStates";
 import { getUrlParams } from "../UrlParams";
 import { type ProcessorState } from "../livekit/TrackProcessorContext";
 import { ElementWidgetActions, widget } from "../widget";
+import { type Async, async, mapAsync, ready } from "./Async";
 
 export interface CallViewModelOptions {
   encryptionSystem: EncryptionSystem;
@@ -449,27 +455,33 @@ export class CallViewModel extends ViewModel {
         }
       : undefined;
 
-  private readonly localFocus = makeFocus(this.matrixRTCSession);
+  private readonly join$ = new Subject<void>();
 
-  private readonly localConnection = this.localFocus.then(
+  public join(): void {
-    (focus) =>
+    this.join$.next();
-      new PublishConnection(
+  }
-        focus,
-        this.livekitAlias,
-        this.matrixRTCSession.room.client,
-        this.scope,
-        this.membershipsAndFocusMap$,
-        this.mediaDevices,
-        this.muteStates,
-        this.e2eeLivekitOptions(),
-        this.scope.behavior(this.trackProcessorState$),
-      ),
-  );
 
-  public readonly livekitConnectionState$ = this.scope.behavior(
+  // This is functionally the same Observable as leave$, except here it's
-    combineLatest([this.localConnection]).pipe(
+  // hoisted to the top of the class. This enables the cyclic dependency between
-      switchMap(([c]) => c.connectionState$),
+  // leave$ -> autoLeave$ -> callPickupState$ -> livekitConnectionState$ ->
-      startWith(ConnectionState.Disconnected),
+  // localConnection$ -> transports$ -> joined$ -> leave$.
+  private readonly leaveHoisted$ = new Subject<
+    "user" | "timeout" | "decline" | "allOthersLeft"
+  >();
+
+  /**
+   * Whether we are joined to the call. This reflects our local state rather
+   * than whether all connections are truly up and running.
+   */
+  private readonly joined$ = this.scope.behavior(
+    this.join$.pipe(
+      map(() => true),
+      // Using takeUntil with the repeat operator is perfectly valid.
+      // eslint-disable-next-line rxjs/no-unsafe-takeuntil
+      takeUntil(this.leaveHoisted$),
+      endWith(false),
+      repeat(),
+      startWith(false),
     ),
   );
 
@@ -488,125 +500,224 @@ export class CallViewModel extends ViewModel {
     ),
   );
 
-  private readonly membershipsAndFocusMap$ = this.scope.behavior(
+  /**
-    this.memberships$.pipe(
+   * The transport that we would personally prefer to publish on (if not for the
-      map((memberships) =>
+   * transport preferences of others, perhaps).
-        memberships.flatMap((m) => {
+   */
-          const f = this.matrixRTCSession.resolveActiveFocus(m);
+  private readonly preferredTransport = makeTransport(this.matrixRTCSession);
-          return f && isLivekitTransport(f)
+
-            ? [{ membership: m, focus: f }]
+  /**
-            : [];
+   * Lists the transports used by ourselves, plus all other MatrixRTC session
-        }),
+   * members.
+   */
+  private readonly transports$: Behavior<{
+    local: Async<LivekitTransport>;
+    remote: { membership: CallMembership; transport: LivekitTransport }[];
+  } | null> = this.scope.behavior(
+    this.joined$.pipe(
+      switchMap((joined) =>
+        joined
+          ? combineLatest(
+              [
+                async(this.preferredTransport),
+                this.memberships$,
+                multiSfu.value$,
+              ],
+              (preferred, memberships, multiSfu) => {
+                const remote = memberships.flatMap((m) => {
+                  if (m.sender === this.userId && m.deviceId === this.deviceId)
+                    return [];
+                  const t = this.matrixRTCSession.resolveActiveFocus(m);
+                  return t && isLivekitTransport(t)
+                    ? [{ membership: m, transport: t }]
+                    : [];
+                });
+                let local = preferred;
+                if (!multiSfu) {
+                  const oldest = this.matrixRTCSession.getOldestMembership();
+                  if (oldest !== undefined) {
+                    const selection = oldest.getTransport(oldest);
+                    if (isLivekitTransport(selection)) local = ready(selection);
+                  }
+                }
+                return { local, remote };
+              },
+            )
+          : of(null),
       ),
     ),
   );
 
-  private readonly livekitServiceUrls$ = this.membershipsAndFocusMap$.pipe(
+  /**
-    map((v) => new Set(v.map(({ focus }) => focus.livekit_service_url))),
+   * Lists the transports used by each MatrixRTC session member other than
+   * ourselves.
+   */
+  private readonly remoteTransports$ = this.scope.behavior(
+    this.transports$.pipe(map((transports) => transports?.remote ?? [])),
   );
 
+  /**
+   * The transport over which we should be actively publishing our media.
+   */
+  private readonly localTransport$: Behavior<Async<LivekitTransport> | null> =
+    this.scope.behavior(
+      this.transports$.pipe(
+        map((transports) => transports?.local ?? null),
+        distinctUntilChanged(deepCompare),
+      ),
+    );
+
+  private readonly localConnectionAndTransport$ = this.scope.behavior(
+    this.localTransport$.pipe(
+      map(
+        (transport) =>
+          transport &&
+          mapAsync(transport, (transport) => ({
+            connection: new PublishConnection(
+              transport,
+              this.livekitAlias,
+              this.matrixRTCSession.room.client,
+              this.scope,
+              this.remoteTransports$,
+              this.mediaDevices,
+              this.muteStates,
+              this.e2eeLivekitOptions(),
+              this.scope.behavior(this.trackProcessorState$),
+            ),
+            transport,
+          })),
+      ),
+    ),
+  );
+
+  private readonly localConnection$ = this.scope.behavior(
+    this.localConnectionAndTransport$.pipe(
+      map((value) => value && mapAsync(value, ({ connection }) => connection)),
+    ),
+  );
+
+  public readonly livekitConnectionState$ = this.scope.behavior(
+    this.localConnection$.pipe(
+      switchMap((c) =>
+        c?.state === "ready"
+          ? c.value.connectionState$
+          : of(ConnectionState.Disconnected),
+      ),
+    ),
+  );
+
+  /**
+   * Connections for each transport in use by one or more session members that
+   * is *distinct* from the local transport.
+   */
   private readonly remoteConnections$ = this.scope.behavior(
-    combineLatest([this.localFocus, this.livekitServiceUrls$]).pipe(
+    this.transports$.pipe(
-      accumulate(
+      accumulate(new Map<string, Connection>(), (prev, transports) => {
-        new Map<string, Connection>(),
+        const next = new Map<string, Connection>();
-        (prev, [localFocus, focusUrls]) => {
-          const stopped = new Map(prev);
-          const next = new Map<string, Connection>();
-          for (const focusUrl of focusUrls) {
-            if (focusUrl !== localFocus.livekit_service_url) {
-              stopped.delete(focusUrl);
 
-              let nextConnection = prev.get(focusUrl);
+        // Until the local transport becomes ready we have no idea which
-              if (!nextConnection) {
+        // transports will actually need a dedicated remote connection
-                logger.log(
+        if (transports?.local.state === "ready") {
-                  "SFU remoteConnections$ construct new connection: ",
+          const localServiceUrl = transports.local.value.livekit_service_url;
-                  focusUrl,
+          const remoteServiceUrls = new Set(
-                );
+            transports.remote.flatMap(({ membership, transport }) => {
-                nextConnection = new Connection(
+              const t = this.matrixRTCSession.resolveActiveFocus(membership);
-                  {
+              return t &&
-                    livekit_service_url: focusUrl,
+                isLivekitTransport(t) &&
-                    livekit_alias: this.livekitAlias,
+                t.livekit_service_url !== localServiceUrl
-                    type: "livekit",
+                ? [t.livekit_service_url]
-                  },
+                : [];
-                  this.livekitAlias,
+            }),
-                  this.matrixRTCSession.room.client,
+          );
-                  this.scope,
+
-                  this.membershipsAndFocusMap$,
+          for (const remoteServiceUrl of remoteServiceUrls) {
-                  this.e2eeLivekitOptions(),
+            let nextConnection = prev.get(remoteServiceUrl);
-                );
+            if (!nextConnection) {
-              } else {
+              logger.log(
-                logger.log(
+                "SFU remoteConnections$ construct new connection: ",
-                  "SFU remoteConnections$ use prev connection: ",
+                remoteServiceUrl,
-                  focusUrl,
+              );
-                );
+              nextConnection = new Connection(
-              }
+                {
-              next.set(focusUrl, nextConnection);
+                  livekit_service_url: remoteServiceUrl,
+                  livekit_alias: this.livekitAlias,
+                  type: "livekit",
+                },
+                this.livekitAlias,
+                this.matrixRTCSession.room.client,
+                this.scope,
+                this.remoteTransports$,
+                this.e2eeLivekitOptions(),
+              );
+            } else {
+              logger.log(
+                "SFU remoteConnections$ use prev connection: ",
+                remoteServiceUrl,
+              );
             }
+            next.set(remoteServiceUrl, nextConnection);
           }
+        }
 
-          for (const connection of stopped.values()) connection.stop();
+        return next;
-          return next;
+      }),
-        },
+      map((transports) => [...transports.values()]),
-      ),
     ),
   );
 
-  private readonly join$ = new Subject<void>();
+  /**
+   * A list of the connections that should be active at any given time.
+   */
+  private readonly connections$ = this.scope.behavior<Connection[]>(
+    combineLatest(
+      [this.localConnection$, this.remoteConnections$],
+      (local, remote) => [
+        ...(local?.state === "ready" ? [local.value] : []),
+        ...remote.values(),
+      ],
+    ),
+  );
 
-  public join(): void {
+  private readonly connectionInstructions$ = this.connections$.pipe(
-    this.join$.next();
-  }
-
-  private readonly connectionInstructions$ = this.join$.pipe(
-    switchMap(() => this.remoteConnections$),
-    startWith(new Map<string, Connection>()),
     pairwise(),
     map(([prev, next]) => {
       const start = new Set(next.values());
-      for (const connection of prev.values()) start.delete(connection);
+      for (const connection of prev) start.delete(connection);
       const stop = new Set(prev.values());
-      for (const connection of next.values()) stop.delete(connection);
+      for (const connection of next) stop.delete(connection);
 
       return { start, stop };
     }),
     this.scope.share,
   );
 
+  /**
+   * Emits with a connection whenever it should be started.
+   */
   private readonly startConnection$ = this.connectionInstructions$.pipe(
     concatMap(({ start }) => start),
   );
+  /**
+   * Emits with a connection whenever it should be stopped.
+   */
   private readonly stopConnection$ = this.connectionInstructions$.pipe(
     concatMap(({ stop }) => stop),
   );
 
   public readonly allLivekitRooms$ = this.scope.behavior(
-    combineLatest([
+    this.connections$.pipe(
-      this.remoteConnections$,
+      map((connections) =>
-      this.localConnection,
+        [...connections.values()].map((c) => ({
-      this.localFocus,
+          room: c.livekitRoom,
-    ]).pipe(
+          url: c.transport.livekit_service_url,
-      map(([remoteConnections, localConnection, localFocus]) =>
+          isLocal: c instanceof PublishConnection,
-        Array.from(remoteConnections.entries())
+        })),
-          .map(
-            ([index, c]) =>
-              ({
-                room: c.livekitRoom,
-                url: index,
-              }) as { room: LivekitRoom; url: string; isLocal?: boolean },
-          )
-          .concat([
-            {
-              room: localConnection.livekitRoom,
-              url: localFocus.livekit_service_url,
-              isLocal: true,
-            },
-          ]),
       ),
-      startWith([]),
     ),
   );
 
   private readonly userId = this.matrixRoom.client.getUserId();
+  private readonly deviceId = this.matrixRoom.client.getDeviceId();
 
   private readonly matrixConnected$ = this.scope.behavior(
     // To consider ourselves connected to MatrixRTC, we check the following:
@@ -679,6 +790,10 @@ export class CallViewModel extends ViewModel {
   // in a split-brained state.
   private readonly pretendToBeDisconnected$ = this.reconnecting$;
 
+  /**
+   * Lists, for each LiveKit room, the LiveKit participants whose media should
+   * be presented.
+   */
   public readonly participantsByRoom$ = this.scope.behavior<
     {
       livekitRoom: LivekitRoom;
@@ -689,9 +804,12 @@ export class CallViewModel extends ViewModel {
       }[];
     }[]
   >(
-    combineLatest([this.localConnection, this.localFocus])
+    // TODO: Move this logic into Connection/PublishConnection if possible
+    this.localConnectionAndTransport$
       .pipe(
-        switchMap(([localConnection, localFocus]) => {
+        switchMap((values) => {
+          if (values?.state !== "ready") return [];
+          const localConnection = values.value.connection;
           const memberError = (): never => {
             throw new Error("No room member for call membership");
           };
@@ -702,12 +820,9 @@ export class CallViewModel extends ViewModel {
           };
 
           return this.remoteConnections$.pipe(
-            switchMap((connections) =>
+            switchMap((remoteConnections) =>
               combineLatest(
-                [
+                [localConnection, ...remoteConnections].map((c) =>
-                  [localFocus.livekit_service_url, localConnection] as const,
-                  ...connections,
-                ].map(([url, c]) =>
                   c.publishingParticipants$.pipe(
                     map((ps) => {
                       const participants: {
@@ -726,7 +841,7 @@ export class CallViewModel extends ViewModel {
 
                       return {
                         livekitRoom: c.livekitRoom,
-                        url,
+                        url: c.transport.livekit_service_url,
                         participants,
                       };
                     }),
@@ -809,12 +924,8 @@ export class CallViewModel extends ViewModel {
    * List of MediaItems that we want to display
    */
   private readonly mediaItems$ = this.scope.behavior<MediaItem[]>(
-    combineLatest([
+    combineLatest([this.participantsByRoom$, duplicateTiles.value$]).pipe(
-      this.participantsByRoom$,
+      scan((prevItems, [participantsByRoom, duplicateTiles]) => {
-      duplicateTiles.value$,
-      this.memberships$,
-    ]).pipe(
-      scan((prevItems, [participantsByRoom, duplicateTiles, memberships]) => {
         const newItems: Map<string, UserMedia | ScreenShare> = new Map(
           function* (this: CallViewModel): Iterable<[string, MediaItem]> {
             for (const { livekitRoom, participants } of participantsByRoom) {
@@ -829,6 +940,7 @@ export class CallViewModel extends ViewModel {
                   if (prevMedia && prevMedia instanceof UserMedia) {
                     prevMedia.updateParticipant(participant);
                     if (prevMedia.vm.member === undefined) {
+                      // TODO-MULTI-SFU: This is outdated.
                       // We have a previous media created because of the `debugShowNonMember` flag.
                       // In this case we actually replace the media item.
                       // This "hack" never occurs if we do not use the `debugShowNonMember` debugging
@@ -931,6 +1043,16 @@ export class CallViewModel extends ViewModel {
     this.memberships$.pipe(map((ms) => ms.length)),
   );
 
+  private readonly allOthersLeft$ = this.memberships$.pipe(
+    pairwise(),
+    filter(
+      ([prev, current]) =>
+        current.every((m) => m.sender === this.userId) &&
+        prev.some((m) => m.sender !== this.userId),
+    ),
+    map(() => {}),
+  );
+
   private readonly didSendCallNotification$ = fromEvent(
     this.matrixRTCSession,
     MatrixRTCSessionEvent.DidSendCallNotification,
@@ -1055,56 +1177,12 @@ export class CallViewModel extends ViewModel {
     map(() => {}),
     throttleTime(THROTTLE_SOUND_EFFECT_MS),
   );
-  /**
-   * This observable tracks the matrix users that are currently in the call.
-   * There can be just one matrix user with multiple participants (see also participantChanges$)
-   */
-  public readonly matrixUserChanges$ = this.userMedia$.pipe(
-    map(
-      (mediaItems) =>
-        new Set(
-          mediaItems
-            .map((m) => m.vm.member?.userId)
-            .filter((id) => id !== undefined),
-        ),
-    ),
-    scan<
-      Set<string>,
-      {
-        userIds: Set<string>;
-        joinedUserIds: Set<string>;
-        leftUserIds: Set<string>;
-      }
-    >(
-      (prevState, userIds) => {
-        const left = new Set(
-          [...prevState.userIds].filter((id) => !userIds.has(id)),
-        );
-        const joined = new Set(
-          [...userIds].filter((id) => !prevState.userIds.has(id)),
-        );
-        return { userIds: userIds, joinedUserIds: joined, leftUserIds: left };
-      },
-      { userIds: new Set(), joinedUserIds: new Set(), leftUserIds: new Set() },
-    ),
-  );
-
-  private readonly allOthersLeft$ = this.matrixUserChanges$.pipe(
-    filter(({ userIds, leftUserIds }) => {
-      if (!this.userId) {
-        logger.warn("Could not access user ID to compute allOthersLeft");
-        return false;
-      }
-      return (
-        userIds.size === 1 && userIds.has(this.userId) && leftUserIds.size > 0
-      );
-    }),
-    map(() => "allOthersLeft" as const),
-  );
 
   // Public for testing
   public readonly autoLeave$ = merge(
-    this.options.autoLeaveWhenOthersLeft ? this.allOthersLeft$ : NEVER,
+    this.options.autoLeaveWhenOthersLeft
+      ? this.allOthersLeft$.pipe(map(() => "allOthersLeft" as const))
+      : NEVER,
     this.callPickupState$.pipe(
       filter((state) => state === "timeout" || state === "decline"),
     ),
@@ -1132,6 +1210,9 @@ export class CallViewModel extends ViewModel {
     merge(this.userHangup$, this.widgetHangup$).pipe(
       map(() => "user" as const),
     ),
+  ).pipe(
+    this.scope.share,
+    tap((reason) => this.leaveHoisted$.next(reason)),
   );
 
   /**
@@ -1820,9 +1901,12 @@ export class CallViewModel extends ViewModel {
    * Whether we are sharing our screen.
    */
   public readonly sharingScreen$ = this.scope.behavior(
-    from(this.localConnection).pipe(
+    from(this.localConnection$).pipe(
-      switchMap((c) => sharingScreen$(c.livekitRoom.localParticipant)),
+      switchMap((c) =>
-      startWith(false),
+        c?.state === "ready"
+          ? sharingScreen$(c.value.livekitRoom.localParticipant)
+          : of(false),
+      ),
     ),
   );
 
@@ -1834,17 +1918,26 @@ export class CallViewModel extends ViewModel {
     "getDisplayMedia" in (navigator.mediaDevices ?? {}) &&
     !this.urlParams.hideScreensharing
       ? (): void =>
-          void this.localConnection.then(
+          // Once a connection is ready...
-            (c) =>
+          void this.localConnection$
-              void c.livekitRoom.localParticipant
+            .pipe(
-                .setScreenShareEnabled(!this.sharingScreen$.value, {
+              takeWhile((c) => c !== null && c.state !== "error"),
-                  audio: true,
+              switchMap((c) => (c.state === "ready" ? of(c.value) : NEVER)),
-                  selfBrowserSurface: "include",
+              take(1),
-                  surfaceSwitching: "include",
+              this.scope.bind(),
-                  systemAudio: "include",
+            )
-                })
+            // ...toggle screen sharing.
-                .catch(logger.error),
+            .subscribe(
-          )
+              (c) =>
+                void c.livekitRoom.localParticipant
+                  .setScreenShareEnabled(!this.sharingScreen$.value, {
+                    audio: true,
+                    selfBrowserSurface: "include",
+                    surfaceSwitching: "include",
+                    systemAudio: "include",
+                  })
+                  .catch(logger.error),
+            )
       : null;
 
   public constructor(
@@ -1864,32 +1957,33 @@ export class CallViewModel extends ViewModel {
   ) {
     super();
 
-    void from(this.localConnection)
+    // Start and stop local and remote connections as needed
-      .pipe(this.scope.bind())
+    this.startConnection$.pipe(this.scope.bind()).subscribe(
-      .subscribe(
+      (c) =>
-        (c) =>
+        void c.start().then(
-          void c
+          () => logger.info(`Connected to ${c.transport.livekit_service_url}`),
-            .start()
+          (e) =>
-            // eslint-disable-next-line no-console
+            logger.error(
-            .then(() => console.log("successfully started publishing"))
+              `Failed to start connection to ${c.transport.livekit_service_url}`,
-            // eslint-disable-next-line no-console
+              e,
-            .catch((e) => console.error("failed to start publishing", e)),
+            ),
-      );
+        ),
+    );
+    this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => {
+      logger.info(`Disconnecting from ${c.transport.livekit_service_url}`);
+      c.stop();
+    });
 
-    this.startConnection$
+    // Start and stop session membership as needed
-      .pipe(this.scope.bind())
+    this.localTransport$.pipe(this.scope.bind()).subscribe((localTransport) => {
-      .subscribe((c) => void c.start());
+      if (localTransport?.state === "ready") {
-    this.stopConnection$.pipe(this.scope.bind()).subscribe((c) => c.stop());
-
-    combineLatest([this.localFocus, this.join$])
-      .pipe(this.scope.bind())
-      .subscribe(([localFocus]) => {
         void enterRTCSession(
           this.matrixRTCSession,
-          localFocus,
+          localTransport.value,
           this.options.encryptionSystem.kind !== E2eeType.NONE,
           true,
           true,
+          multiSfu.value$.value,
         )
           .catch((e) => logger.error("Error entering RTC session", e))
           .then(() =>
@@ -1906,19 +2000,20 @@ export class CallViewModel extends ViewModel {
                   ),
               ),
           );
-      });
 
-    this.leave$.pipe(this.scope.bind()).subscribe(() => {
+        return (): void =>
-      // Only sends Matrix leave event. The LiveKit session will disconnect once, uh...
+          // Only sends Matrix leave event. The LiveKit session will disconnect
-      // (TODO-MULTI-SFU does anything actually cause it to disconnect?)
+          // as soon as either the stopConnection$ handler above gets to it or
-      void this.matrixRTCSession
+          // the view model is destroyed.
-        .leaveRoomSession()
+          void this.matrixRTCSession
-        .catch((e) => logger.error("Error leaving RTC session", e))
+            .leaveRoomSession()
-        .then(async () =>
+            .catch((e) => logger.error("Error leaving RTC session", e))
-          widget?.api.transport
+            .then(async () =>
-            .send(ElementWidgetActions.HangupCall, {})
+              widget?.api.transport
-            .catch((e) => logger.error("Failed to send hangup action", e)),
+                .send(ElementWidgetActions.HangupCall, {})
-        );
+                .catch((e) => logger.error("Failed to send hangup action", e)),
+            );
+      }
     });
 
     // Pause upstream of all local media tracks when we're disconnected from
@@ -1927,10 +2022,12 @@ export class CallViewModel extends ViewModel {
     // We use matrixConnected$ rather than reconnecting$ because we want to
     // pause tracks during the initial joining sequence too until we're sure
     // that our own media is displayed on screen.
-    void this.localConnection.then((localConnection) =>
+    combineLatest([this.localConnection$, this.matrixConnected$])
-      this.matrixConnected$.pipe(this.scope.bind()).subscribe((connected) => {
+      .pipe(this.scope.bind())
+      .subscribe(([connection, connected]) => {
+        if (connection?.state !== "ready") return;
         const publications =
-          localConnection.livekitRoom.localParticipant.trackPublications.values();
+          connection.value.livekitRoom.localParticipant.trackPublications.values();
         if (connected) {
           for (const p of publications) {
             if (p.track?.isUpstreamPaused === true) {
@@ -1966,8 +2063,7 @@ export class CallViewModel extends ViewModel {
             }
           }
         }
-      }),
+      });
-    );
 
     // Join automatically
     this.join(); // TODO-MULTI-SFU: Use this view model for the lobby as well, and only call this once 'join' is clicked?

src/state/Connection.ts

@@ -62,7 +62,7 @@ export class Connection {
 
   protected readonly sfuConfig = getSFUConfigWithOpenID(
     this.client,
-    this.focus.livekit_service_url,
+    this.transport.livekit_service_url,
     this.livekitAlias,
   );
 
@@ -72,12 +72,12 @@ export class Connection {
 
   public connectionState$: Behavior<ConnectionState>;
   public constructor(
-    protected readonly focus: LivekitTransport,
+    public readonly transport: LivekitTransport,
     protected readonly livekitAlias: string,
     protected readonly client: MatrixClient,
     protected readonly scope: ObservableScope,
-    protected readonly membershipsFocusMap$: Behavior<
+    protected readonly remoteTransports$: Behavior<
-      { membership: CallMembership; focus: LivekitTransport }[]
+      { membership: CallMembership; transport: LivekitTransport }[]
     >,
     e2eeLivekitOptions: E2EEOptions | undefined,
     livekitRoom: LivekitRoom | undefined = undefined,
@@ -95,12 +95,13 @@ export class Connection {
 
     this.publishingParticipants$ = this.scope.behavior(
       combineLatest(
-        [this.participantsIncludingSubscribers$, this.membershipsFocusMap$],
+        [this.participantsIncludingSubscribers$, this.remoteTransports$],
-        (participants, membershipsFocusMap) =>
+        (participants, remoteTransports) =>
-          membershipsFocusMap
+          remoteTransports
             // Find all members that claim to publish on this connection
-            .flatMap(({ membership, focus }) =>
+            .flatMap(({ membership, transport }) =>
-              focus.livekit_service_url === this.focus.livekit_service_url
+              transport.livekit_service_url ===
+              this.transport.livekit_service_url
                 ? [membership]
                 : [],
             )
@@ -130,23 +131,35 @@ export class PublishConnection extends Connection {
     if (!this.stopped) await this.livekitRoom.connect(url, jwt);
 
     if (!this.stopped) {
-      const tracks = await this.livekitRoom.localParticipant.createTracks({
+      // TODO-MULTI-SFU: Prepublish a microphone track
-        audio: this.muteStates.audio.enabled$.value,
+      const audio = this.muteStates.audio.enabled$.value;
-        video: this.muteStates.video.enabled$.value,
+      const video = this.muteStates.video.enabled$.value;
-      });
+      // createTracks throws if called with audio=false and video=false
-      for (const track of tracks) {
+      if (audio || video) {
-        await this.livekitRoom.localParticipant.publishTrack(track);
+        const tracks = await this.livekitRoom.localParticipant.createTracks({
+          audio,
+          video,
+        });
+        for (const track of tracks) {
+          await this.livekitRoom.localParticipant.publishTrack(track);
+        }
       }
     }
   }
 
+  public stop(): void {
+    this.muteStates.audio.unsetHandler();
+    this.muteStates.video.unsetHandler();
+    super.stop();
+  }
+
   public constructor(
-    focus: LivekitTransport,
+    transport: LivekitTransport,
     livekitAlias: string,
     client: MatrixClient,
     scope: ObservableScope,
-    membershipsFocusMap$: Behavior<
+    remoteTransports$: Behavior<
-      { membership: CallMembership; focus: LivekitTransport }[]
+      { membership: CallMembership; transport: LivekitTransport }[]
     >,
     devices: MediaDevices,
     private readonly muteStates: MuteStates,
@@ -182,11 +195,11 @@ export class PublishConnection extends Connection {
     });
 
     super(
-      focus,
+      transport,
       livekitAlias,
       client,
       scope,
-      membershipsFocusMap$,
+      remoteTransports$,
       e2eeLivekitOptions,
       room,
     );
@@ -218,10 +231,6 @@ export class PublishConnection extends Connection {
       }
       return this.livekitRoom.localParticipant.isCameraEnabled;
     });
-    this.scope.onEnd(() => {
-      this.muteStates.audio.unsetHandler();
-      this.muteStates.video.unsetHandler();
-    });
 
     const syncDevice = (
       kind: MediaDeviceKind,

src/state/MuteStates.ts

@@ -137,7 +137,7 @@ export class MuteStates {
     this.scope,
     this.mediaDevices.audioInput,
     this.joined$,
-    Config.get().media_devices.enable_video,
+    Config.get().media_devices.enable_audio,
   );
   public readonly video = new MuteState(
     this.scope,

src/state/ObservableScope.ts

@@ -8,9 +8,10 @@ Please see LICENSE in the repository root for full details.
 import {
   BehaviorSubject,
   distinctUntilChanged,
+  filter,
   type Observable,
   share,
-  Subject,
+  take,
   takeUntil,
 } from "rxjs";
 
@@ -24,9 +25,11 @@ const nothing = Symbol("nothing");
  * A scope which limits the execution lifetime of its bound Observables.
  */
 export class ObservableScope {
-  private readonly ended$ = new Subject<void>();
+  private readonly ended$ = new BehaviorSubject(false);
 
-  private readonly bindImpl: MonoTypeOperator = takeUntil(this.ended$);
+  private readonly bindImpl: MonoTypeOperator = takeUntil(
+    this.ended$.pipe(filter((ended) => ended)),
+  );
 
   /**
    * Binds an Observable to this scope, so that it completes when the scope
@@ -78,15 +81,19 @@ export class ObservableScope {
    * Ends the scope, causing any bound Observables to complete.
    */
   public end(): void {
-    this.ended$.next();
+    this.ended$.next(true);
-    this.ended$.complete();
   }
 
   /**
    * Register a callback to be executed when the scope is ended.
    */
   public onEnd(callback: () => void): void {
-    this.ended$.subscribe(callback);
+    this.ended$
+      .pipe(
+        filter((ended) => ended),
+        take(1),
+      )
+      .subscribe(callback);
   }
 }
 

src/utils/errors.ts

@@ -11,7 +11,7 @@ export enum ErrorCode {
   /**
    * Configuration problem due to no MatrixRTC backend/SFU is exposed via .well-known and no fallback configured.
    */
-  MISSING_MATRIX_RTC_FOCUS = "MISSING_MATRIX_RTC_FOCUS",
+  MISSING_MATRIX_RTC_TRANSPORT = "MISSING_MATRIX_RTC_TRANSPORT",
   CONNECTION_LOST_ERROR = "CONNECTION_LOST_ERROR",
   /** LiveKit indicates that the server has hit its track limits */
   INSUFFICIENT_CAPACITY_ERROR = "INSUFFICIENT_CAPACITY_ERROR",
@@ -54,18 +54,18 @@ export class ElementCallError extends Error {
   }
 }
 
-export class MatrixRTCFocusMissingError extends ElementCallError {
+export class MatrixRTCTransportMissingError extends ElementCallError {
   public domain: string;
 
   public constructor(domain: string) {
     super(
       t("error.call_is_not_supported"),
-      ErrorCode.MISSING_MATRIX_RTC_FOCUS,
+      ErrorCode.MISSING_MATRIX_RTC_TRANSPORT,
       ErrorCategory.CONFIGURATION_ISSUE,
-      t("error.matrix_rtc_focus_missing", {
+      t("error.matrix_rtc_transport_missing", {
         domain,
         brand: import.meta.env.VITE_PRODUCT_NAME || "Element Call",
-        errorCode: ErrorCode.MISSING_MATRIX_RTC_FOCUS,
+        errorCode: ErrorCode.MISSING_MATRIX_RTC_TRANSPORT,
       }),
     );
     this.domain = domain;