Make video tiles be based on MatrixRTC member not LiveKit participants (#2701)

* make tiles based on rtc member

* display missing lk participant + fix tile multiplier

* add show_non_member_participants config option

* per member tiles

* merge fixes

* linter

* linter and tests

* tests

* adapt tests (wip)

* Remove unused keys

* Fix optionality of nonMemberItemCount

* video is optional

* Mock RTC members

* Lint

* Merge fixes

* Fix user id

* Add explicit types for public fields

* isRTCParticipantAvailable => isLiveKitParticipantAvailable

* isLiveKitParticipantAvailable

* Readonly

* More keys removal

* Make local field based on view model class not observable

* Wording

* Fix RTC members in tes

* Tests again

* Lint

* Disable showing non-member tiles by default

* Duplicate screen sharing tiles like we used to

* Lint

* Revert function reordering

* Remove throttleTime from bad merge

* Cleanup

* Tidy config of show non-member settings

* tidy up handling of local rtc member in tests

* tidy up test init

* Fix mocks

* Cleanup

* Apply local override where participant not yet known

* Handle no visible media id

* Assertions for one-on-one view

* Remove isLiveKitParticipantAvailable and show via encryption status

* Handle no local media (yet)

* Remove unused effect for setting

* Tidy settings

* Avoid case of one-to-one layout with missing local or remote

* Iterate

* Remove option to show non-member tiles to simplify code review

* Remove unused code

* Remove more remnants of show-non-member-tiles

* iterate

* back

* Fix unit test

* Refactor

* Expose TestScheduler as global

* Fix incorrect type assertion

* Simplify speaking observer

* Fix

* Whitespace

* Make it clear that we are mocking MatrixRTC memberships

* Test case for only showing tiles for MatrixRTC session members

* Simplify diff

* Simplify diff

These changes are in https://github.com/element-hq/element-call/pull/2809

* .

* Whitespaces

* Use asObservable when exposing subject

* Show "waiting for media..." when no participant

* Additional test case

* Don't show "waiting for media..." in case of local participant

* Make the loading state more subtle
 - instead of a label we show a animated gradient

* Use correct key for matrix rtc foci in code comment. (#2838)

* Update src/tile/SpotlightTile.tsx

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Update src/state/CallViewModel.ts

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Make the purpose of BaseMediaViewModel.local explicit

* Use named object instead of unnamed array for spotlightAndPip

* Refactor spotlightAndPip into spotlight and pip

* Use if statement instead of ternary for readability in spotlight and pip logic

* Review feedback

* Fix tests for CallEventAudioRenderer

* Lint

* Revert "Make the loading state more subtle"

This reverts commit 765f7b4f319b86839fcb4fde28d1e0604e542577.

* Update src/state/CallViewModel.ts

Co-authored-by: Timo <16718859+toger5@users.noreply.github.com>

* Fix spelling

* Remove a non-null assertion that failed at runtime

---------

Co-authored-by: Hugh Nimmo-Smith <hughns@element.io>
Co-authored-by: Hugh Nimmo-Smith <hughns@users.noreply.github.com>

locales/en/app.json

@@ -194,6 +194,7 @@
     "expand": "Expand",
     "mute_for_me": "Mute for me",
     "muted_for_me": "Muted for me",
-    "volume": "Volume"
+    "volume": "Volume",
+    "waiting_for_media": "Waiting for media..."
   }
 }

src/config/ConfigOptions.ts

@@ -51,7 +51,7 @@ export interface ConfigOptions {
     // a livekit service url in the client well-known.
     // The well known needs to be formatted like so:
     // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
-    // and stored under the key: "livekit_focus"
+    // and stored under the key: "org.matrix.msc4143.rtc_foci"
     livekit_service_url: string;
   };
 

src/room/CallEventAudioRenderer.test.tsx

@@ -8,10 +8,14 @@ Please see LICENSE in the repository root for full details.
 import { render } from "@testing-library/react";
 import { beforeEach, expect, test } from "vitest";
 import { MatrixClient } from "matrix-js-sdk/src/client";
-import { ConnectionState, RemoteParticipant, Room } from "livekit-client";
+import { ConnectionState, Room } from "livekit-client";
-import { of } from "rxjs";
+import { BehaviorSubject, of } from "rxjs";
 import { afterEach } from "node:test";
 import { act } from "react";
+import {
+  CallMembership,
+  type MatrixRTCSession,
+} from "matrix-js-sdk/src/matrixrtc";
 
 import { soundEffectVolumeSetting } from "../settings/settings";
 import {
@@ -22,6 +26,8 @@ import {
   mockMatrixRoomMember,
   mockMediaPlay,
   mockRemoteParticipant,
+  mockRtcMembership,
+  MockRTCSession,
 } from "../utils/test";
 import { E2eeType } from "../e2ee/e2eeType";
 import { CallViewModel } from "../state/CallViewModel";
@@ -30,11 +36,15 @@ import {
   MAX_PARTICIPANT_COUNT_FOR_SOUND,
 } from "./CallEventAudioRenderer";
 
-const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
-const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
+const local = mockMatrixRoomMember(localRtcMember);
-const aliceId = `${alice.userId}:AAAA`;
+const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
-const bobId = `${bob.userId}:BBBB`;
+const alice = mockMatrixRoomMember(aliceRtcMember);
+const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
+const bob = mockMatrixRoomMember(bobRtcMember);
 const localParticipant = mockLocalParticipant({ identity: "" });
+const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
+const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
 const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
 const bobParticipant = mockRemoteParticipant({ identity: bobId });
 
@@ -53,20 +63,28 @@ afterEach(() => {
 
 test("plays a sound when entering a call", () => {
   const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map(
+    [local, alice, bob].map((p) => [p.userId, p]),
+  );
   const remoteParticipants = of([aliceParticipant]);
   const liveKitRoom = mockLivekitRoom(
     { localParticipant },
     { remoteParticipants },
   );
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(matrixRoom, localRtcMember, [
+    aliceRtcMember,
+  ]) as unknown as MatrixRTCSession;
 
   const vm = new CallViewModel(
-    mockMatrixRoom({
+    session,
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
     liveKitRoom,
     {
       kind: E2eeType.PER_PARTICIPANT,
@@ -84,20 +102,29 @@ test("plays a sound when entering a call", () => {
 test("plays no sound when muted", () => {
   soundEffectVolumeSetting.setValue(0);
   const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice, bob].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map(
+    [local, alice, bob].map((p) => [p.userId, p]),
+  );
   const remoteParticipants = of([aliceParticipant, bobParticipant]);
   const liveKitRoom = mockLivekitRoom(
     { localParticipant },
     { remoteParticipants },
   );
 
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(matrixRoom, localRtcMember, [
+    aliceRtcMember,
+  ]) as unknown as MatrixRTCSession;
+
   const vm = new CallViewModel(
-    mockMatrixRoom({
+    session,
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
     liveKitRoom,
     {
       kind: E2eeType.PER_PARTICIPANT,
@@ -112,7 +139,7 @@ test("plays no sound when muted", () => {
 
 test("plays a sound when a user joins", () => {
   const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
   const remoteParticipants = new Map(
     [aliceParticipant].map((p) => [p.identity, p]),
   );
@@ -121,13 +148,27 @@ test("plays a sound when a user joins", () => {
     remoteParticipants,
   });
 
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
+    aliceRtcMember,
+  ]);
+  // we give Bob an RTC session now, but no participant yet
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(
+    remoteRtcMemberships.asObservable(),
+  ) as unknown as MatrixRTCSession;
+
   const vm = new CallViewModel(
-    mockMatrixRoom({
+    session,
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
     liveKitRoom as unknown as Room,
     {
       kind: E2eeType.PER_PARTICIPANT,
@@ -137,20 +178,20 @@ test("plays a sound when a user joins", () => {
   render(<CallEventAudioRenderer vm={vm} />);
 
   act(() => {
-    liveKitRoom.addParticipant(bobParticipant);
+    remoteRtcMemberships.next([aliceRtcMember, bobRtcMember]);
   });
   // Play a sound when joining a call.
   expect(audioIsPlaying).toEqual([
     // Joining the call
     enterSound,
-    // Bob leaves
+    // Bob joins
     enterSound,
   ]);
 });
 
 test("plays a sound when a user leaves", () => {
   const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
   const remoteParticipants = new Map(
     [aliceParticipant].map((p) => [p.identity, p]),
   );
@@ -159,13 +200,25 @@ test("plays a sound when a user leaves", () => {
     remoteParticipants,
   });
 
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>([
+    aliceRtcMember,
+  ]);
+
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
+
   const vm = new CallViewModel(
-    mockMatrixRoom({
+    session,
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
     liveKitRoom as unknown as Room,
     {
       kind: E2eeType.PER_PARTICIPANT,
@@ -175,7 +228,7 @@ test("plays a sound when a user leaves", () => {
   render(<CallEventAudioRenderer vm={vm} />);
 
   act(() => {
-    liveKitRoom.removeParticipant(aliceParticipant);
+    remoteRtcMemberships.next([]);
   });
   expect(audioIsPlaying).toEqual([
     // Joining the call
@@ -185,30 +238,45 @@ test("plays a sound when a user leaves", () => {
   ]);
 });
 
-test("plays no sound when the participant list", () => {
+test("plays no sound when the session member count is larger than the max, until decreased", () => {
   const audioIsPlaying: string[] = mockMediaPlay();
-  const members = new Map([alice].map((p) => [p.userId, p]));
+  const matrixRoomMembers = new Map([local, alice].map((p) => [p.userId, p]));
-  const remoteParticipants = new Map<string, RemoteParticipant>([
+  const remoteParticipants = new Map(
-    [aliceParticipant.identity, aliceParticipant],
+    [aliceParticipant].map((p) => [p.identity, p]),
-    ...Array.from({ length: MAX_PARTICIPANT_COUNT_FOR_SOUND - 1 }).map<
+  );
-      [string, RemoteParticipant]
+
-    >((_, index) => {
+  const mockRtcMemberships: CallMembership[] = [];
-      const p = mockRemoteParticipant({ identity: `user${index}` });
+
-      return [p.identity, p];
+  for (let i = 0; i < MAX_PARTICIPANT_COUNT_FOR_SOUND; i++) {
-    }),
+    mockRtcMemberships.push(
-  ]);
+      mockRtcMembership(`@user${i}:example.org`, `DEVICE${i}`),
+    );
+  }
+
+  const remoteRtcMemberships = new BehaviorSubject<CallMembership[]>(
+    mockRtcMemberships,
+  );
+
   const liveKitRoom = new EmittableMockLivekitRoom({
     localParticipant,
     remoteParticipants,
   });
 
+  const matrixRoom = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => matrixRoomMembers.get(userId) ?? null,
+  });
+
+  const session = new MockRTCSession(
+    matrixRoom,
+    localRtcMember,
+  ).withMemberships(remoteRtcMemberships) as unknown as MatrixRTCSession;
+
   const vm = new CallViewModel(
-    mockMatrixRoom({
+    session,
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
     liveKitRoom as unknown as Room,
     {
       kind: E2eeType.PER_PARTICIPANT,
@@ -217,9 +285,11 @@ test("plays no sound when the participant list", () => {
   );
   render(<CallEventAudioRenderer vm={vm} />);
   expect(audioIsPlaying).toEqual([]);
-  // When the count drops
+  // When the count drops to the max we should play the leave sound
   act(() => {
-    liveKitRoom.removeParticipant(aliceParticipant);
+    remoteRtcMemberships.next(
+      mockRtcMemberships.slice(0, MAX_PARTICIPANT_COUNT_FOR_SOUND - 1),
+    );
   });
   expect(audioIsPlaying).toEqual([leaveSound]);
 });

src/room/InCallView.tsx

@@ -124,7 +124,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
   useEffect(() => {
     if (livekitRoom !== undefined) {
       const vm = new CallViewModel(
-        props.rtcSession.room,
+        props.rtcSession,
         livekitRoom,
         props.e2eeSystem,
         connStateObservable,
@@ -132,12 +132,7 @@ export const ActiveCall: FC<ActiveCallProps> = (props) => {
       setVm(vm);
       return (): void => vm.destroy();
     }
-  }, [
+  }, [props.rtcSession, livekitRoom, props.e2eeSystem, connStateObservable]);
-    props.rtcSession.room,
-    livekitRoom,
-    props.e2eeSystem,
-    connStateObservable,
-  ]);
 
   if (livekitRoom === undefined || vm === null) return null;
 

src/state/CallViewModel.test.ts

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { test, vi, onTestFinished } from "vitest";
+import { test, vi, onTestFinished, it } from "vitest";
 import {
   combineLatest,
   debounceTime,
@@ -25,6 +25,7 @@ import {
 } from "livekit-client";
 import * as ComponentsCore from "@livekit/components-core";
 import { isEqual } from "lodash-es";
+import { CallMembership, MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc";
 
 import { CallViewModel, Layout } from "./CallViewModel";
 import {
@@ -34,6 +35,8 @@ import {
   mockMatrixRoomMember,
   mockRemoteParticipant,
   withTestScheduler,
+  mockRtcMembership,
+  MockRTCSession,
 } from "../utils/test";
 import {
   ECAddonConnectionState,
@@ -43,14 +46,19 @@ import { E2eeType } from "../e2ee/e2eeType";
 
 vi.mock("@livekit/components-core");
 
-const alice = mockMatrixRoomMember({ userId: "@alice:example.org" });
+const localRtcMember = mockRtcMembership("@carol:example.org", "CCCC");
-const bob = mockMatrixRoomMember({ userId: "@bob:example.org" });
+const aliceRtcMember = mockRtcMembership("@alice:example.org", "AAAA");
-const carol = mockMatrixRoomMember({ userId: "@carol:example.org" });
+const bobRtcMember = mockRtcMembership("@bob:example.org", "BBBB");
-const dave = mockMatrixRoomMember({ userId: "@dave:example.org" });
+const daveRtcMember = mockRtcMembership("@dave:example.org", "DDDD");
 
-const aliceId = `${alice.userId}:AAAA`;
+const alice = mockMatrixRoomMember(aliceRtcMember);
-const bobId = `${bob.userId}:BBBB`;
+const bob = mockMatrixRoomMember(bobRtcMember);
-const daveId = `${dave.userId}:DDDD`;
+const carol = mockMatrixRoomMember(localRtcMember);
+const dave = mockMatrixRoomMember(daveRtcMember);
+
+const aliceId = `${alice.userId}:${aliceRtcMember.deviceId}`;
+const bobId = `${bob.userId}:${bobRtcMember.deviceId}`;
+const daveId = `${dave.userId}:${daveRtcMember.deviceId}`;
 
 const localParticipant = mockLocalParticipant({ identity: "" });
 const aliceParticipant = mockRemoteParticipant({ identity: aliceId });
@@ -65,7 +73,9 @@ const bobSharingScreen = mockRemoteParticipant({
 });
 const daveParticipant = mockRemoteParticipant({ identity: daveId });
 
-const members = new Map([alice, bob, carol, dave].map((p) => [p.userId, p]));
+const roomMembers = new Map(
+  [alice, bob, carol, dave].map((p) => [p.userId, p]),
+);
 
 export interface GridLayoutSummary {
   type: "grid";
@@ -173,10 +183,23 @@ function summarizeLayout(l: Observable<Layout>): Observable<LayoutSummary> {
 
 function withCallViewModel(
   remoteParticipants: Observable<RemoteParticipant[]>,
+  rtcMembers: Observable<Partial<CallMembership>[]>,
   connectionState: Observable<ECConnectionState>,
   speaking: Map<Participant, Observable<boolean>>,
   continuation: (vm: CallViewModel) => void,
 ): void {
+  const room = mockMatrixRoom({
+    client: {
+      getUserId: () => localRtcMember.sender,
+      getDeviceId: () => localRtcMember.deviceId,
+    } as Partial<MatrixClient> as MatrixClient,
+    getMember: (userId) => roomMembers.get(userId) ?? null,
+  });
+  const rtcSession = new MockRTCSession(
+    room,
+    localRtcMember,
+    [],
+  ).withMemberships(rtcMembers);
   const participantsSpy = vi
     .spyOn(ComponentsCore, "connectedParticipantsObserver")
     .mockReturnValue(remoteParticipants);
@@ -209,12 +232,7 @@ function withCallViewModel(
   );
 
   const vm = new CallViewModel(
-    mockMatrixRoom({
+    rtcSession as unknown as MatrixRTCSession,
-      client: {
-        getUserId: () => "@carol:example.org",
-      } as Partial<MatrixClient> as MatrixClient,
-      getMember: (userId) => members.get(userId) ?? null,
-    }),
     liveKitRoom,
     {
       kind: E2eeType.PER_PARTICIPANT,
@@ -247,6 +265,7 @@ test("participants are retained during a focus switch", () => {
         a: [aliceParticipant, bobParticipant],
         b: [],
       }),
+      of([aliceRtcMember, bobRtcMember]),
       hot(connectionInputMarbles, {
         c: ConnectionState.Connected,
         s: ECAddonConnectionState.ECSwitchingFocus,
@@ -288,6 +307,7 @@ test("screen sharing activates spotlight layout", () => {
         c: [aliceSharingScreen, bobSharingScreen],
         d: [aliceParticipant, bobSharingScreen],
       }),
+      of([aliceRtcMember, bobRtcMember]),
       of(ConnectionState.Connected),
       new Map(),
       (vm) => {
@@ -356,7 +376,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
     const modeInputMarbles = "     a";
     // First Bob speaks, then Dave, then Alice
     const aSpeakingInputMarbles = "n- 1998ms - 1999ms y";
-    const bSpeakingInputMarbles = "ny 1998ms n 1999ms ";
+    const bSpeakingInputMarbles = "ny 1998ms n 1999ms -";
     const dSpeakingInputMarbles = "n- 1998ms y 1999ms n";
     // Nothing should change when Bob speaks, because Bob is already on screen.
     // When Dave speaks he should switch with Alice because she's the one who
@@ -366,6 +386,7 @@ test("participants stay in the same order unless to appear/disappear", () => {
 
     withCallViewModel(
       of([aliceParticipant, bobParticipant, daveParticipant]),
+      of([aliceRtcMember, bobRtcMember, daveRtcMember]),
       of(ConnectionState.Connected),
       new Map([
         [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -427,6 +448,7 @@ test("spotlight speakers swap places", () => {
 
     withCallViewModel(
       of([aliceParticipant, bobParticipant, daveParticipant]),
+      of([aliceRtcMember, bobRtcMember, daveRtcMember]),
       of(ConnectionState.Connected),
       new Map([
         [aliceParticipant, hot(aSpeakingInputMarbles, { y: true, n: false })],
@@ -475,6 +497,7 @@ test("layout enters picture-in-picture mode when requested", () => {
 
     withCallViewModel(
       of([aliceParticipant, bobParticipant]),
+      of([aliceRtcMember, bobRtcMember]),
       of(ConnectionState.Connected),
       new Map(),
       (vm) => {
@@ -515,6 +538,7 @@ test("spotlight remembers whether it's expanded", () => {
 
     withCallViewModel(
       of([aliceParticipant, bobParticipant]),
+      of([aliceRtcMember, bobRtcMember]),
       of(ConnectionState.Connected),
       new Map(),
       (vm) => {
@@ -559,3 +583,104 @@ test("spotlight remembers whether it's expanded", () => {
     );
   });
 });
+
+test("participants must have a MatrixRTCSession to be visible", () => {
+  withTestScheduler(({ hot, expectObservable }) => {
+    // iterate through a number of combinations of participants and MatrixRTC memberships
+    // Bob never has an MatrixRTC membership
+    const scenarioInputMarbles = " abcdec";
+    // Bob should never be visible
+    const expectedLayoutMarbles = "a-bc-b";
+
+    withCallViewModel(
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [bobParticipant],
+        c: [aliceParticipant, bobParticipant],
+        d: [aliceParticipant, daveParticipant, bobParticipant],
+        e: [aliceParticipant, daveParticipant, bobSharingScreen],
+      }),
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [],
+        c: [aliceRtcMember],
+        d: [aliceRtcMember, daveRtcMember],
+        e: [aliceRtcMember, daveRtcMember],
+      }),
+      of(ConnectionState.Connected),
+      new Map(),
+      (vm) => {
+        vm.setGridMode("grid");
+        expectObservable(summarizeLayout(vm.layout)).toBe(
+          expectedLayoutMarbles,
+          {
+            a: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0"],
+            },
+            b: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${aliceId}:0`,
+            },
+            c: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
+            },
+          },
+        );
+      },
+    );
+  });
+});
+
+it("should show at least one tile per MatrixRTCSession", () => {
+  withTestScheduler(({ hot, expectObservable }) => {
+    // iterate through some combinations of MatrixRTC memberships
+    const scenarioInputMarbles = " abcd";
+    // There should always be one tile for each MatrixRTCSession
+    const expectedLayoutMarbles = "abcd";
+
+    withCallViewModel(
+      of([]),
+      hot(scenarioInputMarbles, {
+        a: [],
+        b: [aliceRtcMember],
+        c: [aliceRtcMember, daveRtcMember],
+        d: [daveRtcMember],
+      }),
+      of(ConnectionState.Connected),
+      new Map(),
+      (vm) => {
+        vm.setGridMode("grid");
+        expectObservable(summarizeLayout(vm.layout)).toBe(
+          expectedLayoutMarbles,
+          {
+            a: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0"],
+            },
+            b: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${aliceId}:0`,
+            },
+            c: {
+              type: "grid",
+              spotlight: undefined,
+              grid: ["local:0", `${aliceId}:0`, `${daveId}:0`],
+            },
+            d: {
+              type: "one-on-one",
+              local: "local:0",
+              remote: `${daveId}:0`,
+            },
+          },
+        );
+      },
+    );
+  });
+});

src/state/CallViewModel.ts

@@ -18,12 +18,9 @@ import {
   RemoteParticipant,
   Track,
 } from "livekit-client";
+import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
 import {
-  Room as MatrixRoom,
+  BehaviorSubject,
-  RoomMember,
-  RoomStateEvent,
-} from "matrix-js-sdk/src/matrix";
-import {
   EMPTY,
   Observable,
   Subject,
@@ -49,6 +46,10 @@ import {
   withLatestFrom,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/src/logger";
+import {
+  MatrixRTCSession,
+  MatrixRTCSessionEvent,
+} from "matrix-js-sdk/src/matrixrtc";
 
 import { ViewModel } from "./ViewModel";
 import {
@@ -222,41 +223,67 @@ interface LayoutScanState {
 class UserMedia {
   private readonly scope = new ObservableScope();
   public readonly vm: UserMediaViewModel;
+  private readonly participant: BehaviorSubject<
+    LocalParticipant | RemoteParticipant | undefined
+  >;
+
   public readonly speaker: Observable<boolean>;
   public readonly presenter: Observable<boolean>;
-
   public constructor(
     public readonly id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: LocalParticipant | RemoteParticipant | undefined,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
   ) {
-    this.vm = participant.isLocal
+    this.participant = new BehaviorSubject(participant);
-      ? new LocalUserMediaViewModel(
+
-          id,
+    if (participant?.isLocal) {
-          member,
+      this.vm = new LocalUserMediaViewModel(
-          participant as LocalParticipant,
+        this.id,
-          encryptionSystem,
+        member,
-          livekitRoom,
+        this.participant.asObservable() as Observable<LocalParticipant>,
-        )
+        encryptionSystem,
-      : new RemoteUserMediaViewModel(
+        livekitRoom,
-          id,
+      );
-          member,
+    } else {
-          participant as RemoteParticipant,
+      this.vm = new RemoteUserMediaViewModel(
-          encryptionSystem,
+        id,
-          livekitRoom,
+        member,
-        );
+        this.participant.asObservable() as Observable<
+          RemoteParticipant | undefined
+        >,
+        encryptionSystem,
+        livekitRoom,
+      );
+    }
 
     this.speaker = observeSpeaker(this.vm.speaking).pipe(this.scope.state());
 
-    this.presenter = observeParticipantEvents(
+    this.presenter = this.participant.pipe(
-      participant,
+      switchMap(
-      ParticipantEvent.TrackPublished,
+        (p) =>
-      ParticipantEvent.TrackUnpublished,
+          (p &&
-      ParticipantEvent.LocalTrackPublished,
+            observeParticipantEvents(
-      ParticipantEvent.LocalTrackUnpublished,
+              p,
-    ).pipe(map((p) => p.isScreenShareEnabled));
+              ParticipantEvent.TrackPublished,
+              ParticipantEvent.TrackUnpublished,
+              ParticipantEvent.LocalTrackPublished,
+              ParticipantEvent.LocalTrackUnpublished,
+            ).pipe(map((p) => p.isScreenShareEnabled))) ??
+          of(false),
+      ),
+      this.scope.state(),
+    );
+  }
+
+  public updateParticipant(
+    newParticipant: LocalParticipant | RemoteParticipant | undefined,
+  ): void {
+    if (this.participant.value !== newParticipant) {
+      // Update the BehaviourSubject in the UserMedia.
+      this.participant.next(newParticipant);
+    }
   }
 
   public destroy(): void {
@@ -267,6 +294,9 @@ class UserMedia {
 
 class ScreenShare {
   public readonly vm: ScreenShareViewModel;
+  private readonly participant: BehaviorSubject<
+    LocalParticipant | RemoteParticipant
+  >;
 
   public constructor(
     id: string,
@@ -275,12 +305,15 @@ class ScreenShare {
     encryptionSystem: EncryptionSystem,
     liveKitRoom: LivekitRoom,
   ) {
+    this.participant = new BehaviorSubject(participant);
+
     this.vm = new ScreenShareViewModel(
       id,
       member,
-      participant,
+      this.participant.asObservable(),
       encryptionSystem,
       liveKitRoom,
+      participant.isLocal,
     );
   }
 
@@ -317,11 +350,11 @@ function findMatrixRoomMember(
 export class CallViewModel extends ViewModel {
   public readonly localVideo: Observable<LocalVideoTrack | null> =
     observeTrackReference(
-      this.livekitRoom.localParticipant,
+      of(this.livekitRoom.localParticipant),
       Track.Source.Camera,
     ).pipe(
       map((trackRef) => {
-        const track = trackRef.publication?.track;
+        const track = trackRef?.publication?.track;
         return track instanceof LocalVideoTrack ? track : null;
       }),
     );
@@ -401,49 +434,87 @@ export class CallViewModel extends ViewModel {
     this.remoteParticipants,
     observeParticipantMedia(this.livekitRoom.localParticipant),
     duplicateTiles.value,
-    // Also react to changes in the list of members
+    // Also react to changes in the MatrixRTC session list.
-    fromEvent(this.matrixRoom, RoomStateEvent.Update).pipe(startWith(null)),
+    // The session list will also be update if a room membership changes.
+    // No additional RoomState event listener needs to be set up.
+    fromEvent(
+      this.matrixRTCSession,
+      MatrixRTCSessionEvent.MembershipsChanged,
+    ).pipe(startWith(null)),
   ]).pipe(
     scan(
       (
         prevItems,
-        [remoteParticipants, { participant: localParticipant }, duplicateTiles],
+        [
+          remoteParticipants,
+          { participant: localParticipant },
+          duplicateTiles,
+          _membershipsChanged,
+        ],
       ) => {
         const newItems = new Map(
           function* (this: CallViewModel): Iterable<[string, MediaItem]> {
-            for (const p of [localParticipant, ...remoteParticipants]) {
+            // m.rtc.members are the basis for calculating what is visible in the call
-              const id = p === localParticipant ? "local" : p.identity;
+            for (const rtcMember of this.matrixRTCSession.memberships) {
-              const member = findMatrixRoomMember(this.matrixRoom, id);
+              const room = this.matrixRTCSession.room;
-              if (member === undefined)
+              // WARN! This is not exactly the sender but the user defined in the state key.
-                logger.warn(
+              // This will be available once we change to the new "member as object" format in the MatrixRTC object.
-                  `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
+              let livekitParticipantId =
-                );
+                rtcMember.sender + ":" + rtcMember.deviceId;
 
-              // Create as many tiles for this participant as called for by
+              let participant:
-              // the duplicateTiles option
+                | LocalParticipant
+                | RemoteParticipant
+                | undefined = undefined;
+              if (
+                rtcMember.sender === room.client.getUserId()! &&
+                rtcMember.deviceId === room.client.getDeviceId()
+              ) {
+                livekitParticipantId = "local";
+                participant = localParticipant;
+              } else {
+                participant = remoteParticipants.find(
+                  (p) => p.identity === livekitParticipantId,
+                );
+              }
+
+              const member = findMatrixRoomMember(room, livekitParticipantId);
+              if (!member) {
+                logger.error(
+                  "Could not find member for media id: ",
+                  livekitParticipantId,
+                );
+              }
               for (let i = 0; i < 1 + duplicateTiles; i++) {
-                const userMediaId = `${id}:${i}`;
+                const indexedMediaId = `${livekitParticipantId}:${i}`;
+                const prevMedia = prevItems.get(indexedMediaId);
+                if (prevMedia && prevMedia instanceof UserMedia) {
+                  prevMedia.updateParticipant(participant);
+                }
                 yield [
-                  userMediaId,
+                  indexedMediaId,
-                  prevItems.get(userMediaId) ??
+                  // We create UserMedia with or without a participant.
+                  // This will be the initial value of a BehaviourSubject.
+                  // Once a participant appears we will update the BehaviourSubject. (see above)
+                  prevMedia ??
                     new UserMedia(
-                      userMediaId,
+                      indexedMediaId,
                       member,
-                      p,
+                      participant,
                       this.encryptionSystem,
                       this.livekitRoom,
                     ),
                 ];
 
-                if (p.isScreenShareEnabled) {
+                if (participant?.isScreenShareEnabled) {
-                  const screenShareId = `${userMediaId}:screen-share`;
+                  const screenShareId = `${indexedMediaId}:screen-share`;
                   yield [
                     screenShareId,
                     prevItems.get(screenShareId) ??
                       new ScreenShare(
                         screenShareId,
                         member,
-                        p,
+                        participant,
                         this.encryptionSystem,
                         this.livekitRoom,
                       ),
@@ -454,7 +525,6 @@ export class CallViewModel extends ViewModel {
           }.bind(this)(),
         );
 
-        for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
         return newItems;
       },
       new Map<string, MediaItem>(),
@@ -488,11 +558,6 @@ export class CallViewModel extends ViewModel {
       ),
     );
 
-  private readonly localUserMedia: Observable<LocalUserMediaViewModel> =
-    this.mediaItems.pipe(
-      map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel),
-    );
-
   /**
    * List of MediaItems that we want to display, that are of type ScreenShare
    */
@@ -504,7 +569,7 @@ export class CallViewModel extends ViewModel {
       this.scope.state(),
     );
 
-  private readonly spotlightSpeaker: Observable<UserMediaViewModel> =
+  private readonly spotlightSpeaker: Observable<UserMediaViewModel | null> =
     this.userMedia.pipe(
       switchMap((mediaItems) =>
         mediaItems.length === 0
@@ -515,7 +580,7 @@ export class CallViewModel extends ViewModel {
               ),
             ),
       ),
-      scan<(readonly [UserMedia, boolean])[], UserMedia, null>(
+      scan<(readonly [UserMedia, boolean])[], UserMedia | undefined, null>(
         (prev, mediaItems) => {
           // Only remote users that are still in the call should be sticky
           const [stickyMedia, stickySpeaking] =
@@ -532,11 +597,11 @@ export class CallViewModel extends ViewModel {
                 // Otherwise, spotlight an arbitrary remote user
                 mediaItems.find(([m]) => !m.vm.local)?.[0] ??
                 // Otherwise, spotlight the local user
-                mediaItems.find(([m]) => m.vm.local)![0]);
+                mediaItems.find(([m]) => m.vm.local)?.[0]);
         },
         null,
       ),
-      map((speaker) => speaker.vm),
+      map((speaker) => speaker?.vm ?? null),
       this.scope.state(),
     );
 
@@ -576,37 +641,57 @@ export class CallViewModel extends ViewModel {
     }),
   );
 
-  private readonly spotlightAndPip: Observable<
-    [Observable<MediaViewModel[]>, Observable<UserMediaViewModel | null>]
-  > = this.screenShares.pipe(
-    map((screenShares) =>
-      screenShares.length > 0
-        ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const)
-        : ([
-            this.spotlightSpeaker.pipe(map((speaker) => [speaker!])),
-            this.spotlightSpeaker.pipe(
-              switchMap((speaker) =>
-                speaker.local
-                  ? of(null)
-                  : this.localUserMedia.pipe(
-                      switchMap((vm) =>
-                        vm.alwaysShow.pipe(
-                          map((alwaysShow) => (alwaysShow ? vm : null)),
-                        ),
-                      ),
-                    ),
-              ),
-            ),
-          ] as const),
-    ),
-  );
-
   private readonly spotlight: Observable<MediaViewModel[]> =
-    this.spotlightAndPip.pipe(
+    this.screenShares.pipe(
-      switchMap(([spotlight]) => spotlight),
+      switchMap((screenShares) => {
+        if (screenShares.length > 0) {
+          return of(screenShares.map((m) => m.vm));
+        }
+
+        return this.spotlightSpeaker.pipe(
+          map((speaker) => (speaker ? [speaker] : [])),
+        );
+      }),
       this.scope.state(),
     );
 
+  private readonly pip: Observable<UserMediaViewModel | null> = combineLatest([
+    this.screenShares,
+    this.spotlightSpeaker,
+    this.mediaItems,
+  ]).pipe(
+    switchMap(([screenShares, spotlight, mediaItems]) => {
+      if (screenShares.length > 0) {
+        return this.spotlightSpeaker;
+      }
+      if (!spotlight || spotlight.local) {
+        return of(null);
+      }
+
+      const localUserMedia = mediaItems.find(
+        (m) => m.vm instanceof LocalUserMediaViewModel,
+      ) as UserMedia | undefined;
+
+      const localUserMediaViewModel = localUserMedia?.vm as
+        | LocalUserMediaViewModel
+        | undefined;
+
+      if (!localUserMediaViewModel) {
+        return of(null);
+      }
+      return localUserMediaViewModel.alwaysShow.pipe(
+        map((alwaysShow) => {
+          if (alwaysShow) {
+            return localUserMediaViewModel;
+          }
+
+          return null;
+        }),
+      );
+    }),
+    this.scope.state(),
+  );
+
   private readonly hasRemoteScreenShares: Observable<boolean> =
     this.spotlight.pipe(
       map((spotlight) =>
@@ -615,9 +700,6 @@ export class CallViewModel extends ViewModel {
       distinctUntilChanged(),
     );
 
-  private readonly pip: Observable<UserMediaViewModel | null> =
-    this.spotlightAndPip.pipe(switchMap(([, pip]) => pip));
-
   private readonly pipEnabled: Observable<boolean> = setPipEnabled.pipe(
     startWith(false),
   );
@@ -721,15 +803,16 @@ export class CallViewModel extends ViewModel {
     this.mediaItems.pipe(
       map((mediaItems) => {
         if (mediaItems.length !== 2) return null;
-        const local = mediaItems.find((vm) => vm.vm.local)!
+        const local = mediaItems.find((vm) => vm.vm.local)?.vm as
-          .vm as LocalUserMediaViewModel;
+          | LocalUserMediaViewModel
+          | undefined;
         const remote = mediaItems.find((vm) => !vm.vm.local)?.vm as
           | RemoteUserMediaViewModel
           | undefined;
         // There might not be a remote tile if there are screen shares, or if
         // only the local user is in the call and they're using the duplicate
         // tiles option
-        if (remote === undefined) return null;
+        if (!remote || !local) return null;
 
         return { type: "one-on-one", local, remote };
       }),
@@ -1010,7 +1093,7 @@ export class CallViewModel extends ViewModel {
 
   public constructor(
     // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly matrixRTCSession: MatrixRTCSession,
     private readonly livekitRoom: LivekitRoom,
     private readonly encryptionSystem: EncryptionSystem,
     private readonly connectionState: Observable<ECConnectionState>,

src/state/MediaViewModel.test.ts

@@ -8,14 +8,17 @@ Please see LICENSE in the repository root for full details.
 import { expect, test, vi } from "vitest";
 
 import {
+  mockRtcMembership,
   withLocalMedia,
   withRemoteMedia,
   withTestScheduler,
 } from "../utils/test";
 
+const rtcMembership = mockRtcMembership("@alice:example.org", "AAAA");
+
 test("control a participant's volume", async () => {
   const setVolumeSpy = vi.fn();
-  await withRemoteMedia({}, { setVolume: setVolumeSpy }, (vm) =>
+  await withRemoteMedia(rtcMembership, {}, { setVolume: setVolumeSpy }, (vm) =>
     withTestScheduler(({ expectObservable, schedule }) => {
       schedule("-ab---c---d|", {
         a() {
@@ -60,7 +63,7 @@ test("control a participant's volume", async () => {
 });
 
 test("toggle fit/contain for a participant's video", async () => {
-  await withRemoteMedia({}, {}, (vm) =>
+  await withRemoteMedia(rtcMembership, {}, {}, (vm) =>
     withTestScheduler(({ expectObservable, schedule }) => {
       schedule("-ab|", {
         a: () => vm.toggleFitContain(),
@@ -76,17 +79,21 @@ test("toggle fit/contain for a participant's video", async () => {
 });
 
 test("local media remembers whether it should always be shown", async () => {
-  await withLocalMedia({}, (vm) =>
+  await withLocalMedia(rtcMembership, {}, (vm) =>
     withTestScheduler(({ expectObservable, schedule }) => {
       schedule("-a|", { a: () => vm.setAlwaysShow(false) });
       expectObservable(vm.alwaysShow).toBe("ab", { a: true, b: false });
     }),
   );
   // Next local media should start out *not* always shown
-  await withLocalMedia({}, (vm) =>
+  await withLocalMedia(
-    withTestScheduler(({ expectObservable, schedule }) => {
+    rtcMembership,
-      schedule("-a|", { a: () => vm.setAlwaysShow(true) });
+
-      expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
+    {},
-    }),
+    (vm) =>
+      withTestScheduler(({ expectObservable, schedule }) => {
+        schedule("-a|", { a: () => vm.setAlwaysShow(true) });
+        expectObservable(vm.alwaysShow).toBe("ab", { a: false, b: true });
+      }),
   );
 });

src/state/MediaViewModel.ts

@@ -32,7 +32,6 @@ import {
   Observable,
   Subject,
   combineLatest,
-  distinctUntilChanged,
   distinctUntilKeyChanged,
   filter,
   fromEvent,
@@ -40,7 +39,6 @@ import {
   map,
   merge,
   of,
-  shareReplay,
   startWith,
   switchMap,
   throttleTime,
@@ -77,16 +75,24 @@ export function useDisplayName(vm: MediaViewModel): string {
 }
 
 export function observeTrackReference(
-  participant: Participant,
+  participant: Observable<Participant | undefined>,
   source: Track.Source,
-): Observable<TrackReferenceOrPlaceholder> {
+): Observable<TrackReferenceOrPlaceholder | undefined> {
-  return observeParticipantMedia(participant).pipe(
+  return participant.pipe(
-    map(() => ({
+    switchMap((p) => {
-      participant,
+      if (p) {
-      publication: participant.getTrackPublication(source),
+        return observeParticipantMedia(p).pipe(
-      source,
+          map(() => ({
-    })),
+            participant: p,
-    distinctUntilKeyChanged("publication"),
+            publication: p.getTrackPublication(source),
+            source,
+          })),
+          distinctUntilKeyChanged("publication"),
+        );
+      } else {
+        return of(undefined);
+      }
+    }),
   );
 }
 
@@ -105,11 +111,11 @@ function observeRemoteTrackReceivingOkay(
   };
 
   return combineLatest([
-    observeTrackReference(participant, source),
+    observeTrackReference(of(participant), source),
     interval(1000).pipe(startWith(0)),
   ]).pipe(
     switchMap(async ([trackReference]) => {
-      const track = trackReference.publication?.track;
+      const track = trackReference?.publication?.track;
       if (!track || !(track instanceof RemoteTrack)) {
         return undefined;
       }
@@ -200,14 +206,10 @@ export enum EncryptionStatus {
 }
 
 abstract class BaseMediaViewModel extends ViewModel {
-  /**
-   * Whether the media belongs to the local user.
-   */
-  public readonly local = this.participant.isLocal;
   /**
    * The LiveKit video track for this media.
    */
-  public readonly video: Observable<TrackReferenceOrPlaceholder>;
+  public readonly video: Observable<TrackReferenceOrPlaceholder | undefined>;
   /**
    * Whether there should be a warning that this media is unencrypted.
    */
@@ -215,6 +217,11 @@ abstract class BaseMediaViewModel extends ViewModel {
 
   public readonly encryptionStatus: Observable<EncryptionStatus>;
 
+  /**
+   * Whether this media corresponds to the local participant.
+   */
+  public abstract readonly local: boolean;
+
   public constructor(
     /**
      * An opaque identifier for this media.
@@ -226,7 +233,12 @@ abstract class BaseMediaViewModel extends ViewModel {
     // TODO: Fully separate the data layer from the UI layer by keeping the
     // member object internal
     public readonly member: RoomMember | undefined,
-    protected readonly participant: LocalParticipant | RemoteParticipant,
+    // We don't necessarily have a participant if a user connects via MatrixRTC but not (yet) through
+    // livekit.
+    protected readonly participant: Observable<
+      LocalParticipant | RemoteParticipant | undefined
+    >,
+
     encryptionSystem: EncryptionSystem,
     audioSource: AudioSource,
     videoSource: VideoSource,
@@ -243,69 +255,72 @@ abstract class BaseMediaViewModel extends ViewModel {
       [audio, this.video],
       (a, v) =>
         encryptionSystem.kind !== E2eeType.NONE &&
-        (a.publication?.isEncrypted === false ||
+        (a?.publication?.isEncrypted === false ||
-          v.publication?.isEncrypted === false),
+          v?.publication?.isEncrypted === false),
-    ).pipe(
+    ).pipe(this.scope.state());
-      distinctUntilChanged(),
-      shareReplay({ bufferSize: 1, refCount: false }),
-    );
 
-    if (participant.isLocal || encryptionSystem.kind === E2eeType.NONE) {
+    this.encryptionStatus = this.participant.pipe(
-      this.encryptionStatus = of(EncryptionStatus.Okay).pipe(
+      switchMap((participant): Observable<EncryptionStatus> => {
-        this.scope.state(),
+        if (!participant) {
-      );
+          return of(EncryptionStatus.Connecting);
-    } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
+        } else if (
-      this.encryptionStatus = combineLatest([
+          participant.isLocal ||
-        encryptionErrorObservable(
+          encryptionSystem.kind === E2eeType.NONE
-          livekitRoom,
+        ) {
-          participant,
+          return of(EncryptionStatus.Okay);
-          encryptionSystem,
+        } else if (encryptionSystem.kind === E2eeType.PER_PARTICIPANT) {
-          "MissingKey",
+          return combineLatest([
-        ),
+            encryptionErrorObservable(
-        encryptionErrorObservable(
+              livekitRoom,
-          livekitRoom,
+              participant,
-          participant,
+              encryptionSystem,
-          encryptionSystem,
+              "MissingKey",
-          "InvalidKey",
+            ),
-        ),
+            encryptionErrorObservable(
-        observeRemoteTrackReceivingOkay(participant, audioSource),
+              livekitRoom,
-        observeRemoteTrackReceivingOkay(participant, videoSource),
+              participant,
-      ]).pipe(
+              encryptionSystem,
-        map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
+              "InvalidKey",
-          if (keyMissing) return EncryptionStatus.KeyMissing;
+            ),
-          if (keyInvalid) return EncryptionStatus.KeyInvalid;
+            observeRemoteTrackReceivingOkay(participant, audioSource),
-          if (audioOkay || videoOkay) return EncryptionStatus.Okay;
+            observeRemoteTrackReceivingOkay(participant, videoSource),
-          return undefined; // no change
+          ]).pipe(
-        }),
+            map(([keyMissing, keyInvalid, audioOkay, videoOkay]) => {
-        filter((x) => !!x),
+              if (keyMissing) return EncryptionStatus.KeyMissing;
-        startWith(EncryptionStatus.Connecting),
+              if (keyInvalid) return EncryptionStatus.KeyInvalid;
-        this.scope.state(),
+              if (audioOkay || videoOkay) return EncryptionStatus.Okay;
-      );
+              return undefined; // no change
-    } else {
+            }),
-      this.encryptionStatus = combineLatest([
+            filter((x) => !!x),
-        encryptionErrorObservable(
+            startWith(EncryptionStatus.Connecting),
-          livekitRoom,
+          );
-          participant,
+        } else {
-          encryptionSystem,
+          return combineLatest([
-          "InvalidKey",
+            encryptionErrorObservable(
-        ),
+              livekitRoom,
-        observeRemoteTrackReceivingOkay(participant, audioSource),
+              participant,
-        observeRemoteTrackReceivingOkay(participant, videoSource),
+              encryptionSystem,
-      ]).pipe(
+              "InvalidKey",
-        map(
+            ),
-          ([keyInvalid, audioOkay, videoOkay]):
+            observeRemoteTrackReceivingOkay(participant, audioSource),
-            | EncryptionStatus
+            observeRemoteTrackReceivingOkay(participant, videoSource),
-            | undefined => {
+          ]).pipe(
-            if (keyInvalid) return EncryptionStatus.PasswordInvalid;
+            map(
-            if (audioOkay || videoOkay) return EncryptionStatus.Okay;
+              ([keyInvalid, audioOkay, videoOkay]):
-            return undefined; // no change
+                | EncryptionStatus
-          },
+                | undefined => {
-        ),
+                if (keyInvalid) return EncryptionStatus.PasswordInvalid;
-        filter((x) => !!x),
+                if (audioOkay || videoOkay) return EncryptionStatus.Okay;
-        startWith(EncryptionStatus.Connecting),
+                return undefined; // no change
-        this.scope.state(),
+              },
-      );
+            ),
-    }
+            filter((x) => !!x),
+            startWith(EncryptionStatus.Connecting),
+          );
+        }
+      }),
+      this.scope.state(),
+    );
   }
 }
 
@@ -324,11 +339,14 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   /**
    * Whether the participant is speaking.
    */
-  public readonly speaking = observeParticipantEvents(
+  public readonly speaking = this.participant.pipe(
-    this.participant,
+    switchMap((p) =>
-    ParticipantEvent.IsSpeakingChanged,
+      p
-  ).pipe(
+        ? observeParticipantEvents(p, ParticipantEvent.IsSpeakingChanged).pipe(
-    map((p) => p.isSpeaking),
+            map((p) => p.isSpeaking),
+          )
+        : of(false),
+    ),
     this.scope.state(),
   );
 
@@ -350,7 +368,7 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
   ) {
@@ -364,18 +382,25 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel {
       livekitRoom,
     );
 
-    const media = observeParticipantMedia(participant).pipe(this.scope.state());
+    const media = participant.pipe(
+      switchMap((p) => (p && observeParticipantMedia(p)) ?? of(undefined)),
+      this.scope.state(),
+    );
     this.audioEnabled = media.pipe(
-      map((m) => m.microphoneTrack?.isMuted === false),
+      map((m) => m?.microphoneTrack?.isMuted === false),
     );
     this.videoEnabled = media.pipe(
-      map((m) => m.cameraTrack?.isMuted === false),
+      map((m) => m?.cameraTrack?.isMuted === false),
     );
   }
 
   public toggleFitContain(): void {
     this._cropVideo.next(!this._cropVideo.value);
   }
+
+  public get local(): boolean {
+    return this instanceof LocalUserMediaViewModel;
+  }
 }
 
 /**
@@ -387,7 +412,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
    */
   public readonly mirror = this.video.pipe(
     switchMap((v) => {
-      const track = v.publication?.track;
+      const track = v?.publication?.track;
       if (!(track instanceof LocalTrack)) return of(false);
       // Watch for track restarts, because they indicate a camera switch
       return fromEvent(track, TrackEvent.Restarted).pipe(
@@ -409,7 +434,7 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant,
+    participant: Observable<LocalParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
   ) {
@@ -470,18 +495,17 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: RemoteParticipant,
+    participant: Observable<RemoteParticipant | undefined>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
   ) {
     super(id, member, participant, encryptionSystem, livekitRoom);
 
     // Sync the local volume with LiveKit
-    this.localVolume
+    combineLatest([
-      .pipe(this.scope.bind())
+      participant,
-      .subscribe((volume) =>
+      this.localVolume.pipe(this.scope.bind()),
-        (this.participant as RemoteParticipant).setVolume(volume),
+    ]).subscribe(([p, volume]) => p && p.setVolume(volume));
-      );
   }
 
   public toggleLocallyMuted(): void {
@@ -504,9 +528,10 @@ export class ScreenShareViewModel extends BaseMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,
-    participant: LocalParticipant | RemoteParticipant,
+    participant: Observable<LocalParticipant | RemoteParticipant>,
     encryptionSystem: EncryptionSystem,
     livekitRoom: LivekitRoom,
+    public readonly local: boolean,
   ) {
     super(
       id,

src/tile/GridTile.test.tsx

@@ -13,7 +13,7 @@ import { of } from "rxjs";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
 
 import { GridTile } from "./GridTile";
-import { withRemoteMedia } from "../utils/test";
+import { mockRtcMembership, withRemoteMedia } from "../utils/test";
 import { GridTileViewModel } from "../state/TileViewModel";
 import { ReactionsProvider } from "../useReactions";
 
@@ -25,6 +25,7 @@ global.IntersectionObserver = class MockIntersectionObserver {
 
 test("GridTile is accessible", async () => {
   await withRemoteMedia(
+    mockRtcMembership("@alice:example.org", "AAAA"),
     {
       rawDisplayName: "Alice",
       getMxcAvatarUrl: () => "mxc://adfsg",

src/tile/GridTile.tsx

@@ -175,6 +175,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
         raisedHandTime={handRaised}
         currentReaction={currentReaction}
         raisedHandOnClick={raisedHandOnClick}
+        localParticipant={vm.local}
         {...props}
       />
     );

src/tile/MediaView.module.css

@@ -74,9 +74,9 @@ unconditionally select the container so we can use cqmin units */
     calc(var(--media-view-border-radius) - var(--cpd-space-3x))
   );
   display: grid;
-  grid-template-columns: 1fr auto;
+  grid-template-columns: 30px 1fr 30px;
   grid-template-rows: 1fr auto;
-  grid-template-areas: "status status" "nameTag button";
+  grid-template-areas: "reactions status ." "nameTag nameTag button";
   gap: var(--cpd-space-1x);
   place-items: start;
 }
@@ -101,8 +101,8 @@ unconditionally select the container so we can use cqmin units */
   grid-area: status;
   justify-self: center;
   align-self: start;
-  padding: var(--cpd-space-1x);
+  padding: var(--cpd-space-2x);
-  padding-block: var(--cpd-space-1x);
+  padding-block: var(--cpd-space-2x);
   color: var(--cpd-color-text-primary);
   background-color: var(--cpd-color-bg-canvas-default);
   display: flex;
@@ -116,6 +116,12 @@ unconditionally select the container so we can use cqmin units */
   text-align: center;
 }
 
+.reactions {
+  grid-area: reactions;
+  display: flex;
+  gap: var(--cpd-space-1x);
+}
+
 .nameTag > svg,
 .nameTag > span {
   flex-shrink: 0;

src/tile/MediaView.test.tsx

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only
 Please see LICENSE in the repository root for full details.
 */
 
-import { describe, expect, test } from "vitest";
+import { describe, expect, it, test } from "vitest";
 import { render, screen } from "@testing-library/react";
 import { axe } from "vitest-axe";
 import { TooltipProvider } from "@vector-im/compound-web";
@@ -42,6 +42,7 @@ describe("MediaView", () => {
     unencryptedWarning: false,
     video: trackReference,
     member: undefined,
+    localParticipant: false,
   };
 
   test("is accessible", async () => {
@@ -59,6 +60,25 @@ describe("MediaView", () => {
     });
   });
 
+  describe("with no participant", () => {
+    it("shows avatar for local user", () => {
+      render(
+        <MediaView {...baseProps} video={undefined} localParticipant={true} />,
+      );
+      expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
+      expect(screen.queryAllByText("video_tile.waiting_for_media").length).toBe(
+        0,
+      );
+    });
+    it("shows avatar and label for remote user", () => {
+      render(
+        <MediaView {...baseProps} video={undefined} localParticipant={false} />,
+      );
+      expect(screen.getByRole("img", { name: "some name" })).toBeVisible();
+      expect(screen.getByText("video_tile.waiting_for_media")).toBeVisible();
+    });
+  });
+
   describe("name tag", () => {
     test("is shown with name", () => {
       render(<MediaView {...baseProps} displayName="Bob" />);

src/tile/MediaView.tsx

@@ -28,7 +28,7 @@ interface Props extends ComponentProps<typeof animated.div> {
   style?: ComponentProps<typeof animated.div>["style"];
   targetWidth: number;
   targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video: TrackReferenceOrPlaceholder | undefined;
   videoFit: "cover" | "contain";
   mirror: boolean;
   member: RoomMember | undefined;
@@ -41,6 +41,7 @@ interface Props extends ComponentProps<typeof animated.div> {
   raisedHandTime?: Date;
   currentReaction?: ReactionOption;
   raisedHandOnClick?: () => void;
+  localParticipant: boolean;
 }
 
 export const MediaView = forwardRef<HTMLDivElement, Props>(
@@ -63,6 +64,7 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
       raisedHandTime,
       currentReaction,
       raisedHandOnClick,
+      localParticipant,
       ...props
     },
     ref,
@@ -90,21 +92,21 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
             size={avatarSize}
             src={member?.getMxcAvatarUrl()}
             className={styles.avatar}
-            style={{ display: videoEnabled ? "none" : "initial" }}
+            style={{ display: video && videoEnabled ? "none" : "initial" }}
           />
-          {video.publication !== undefined && (
+          {video?.publication !== undefined && (
             <VideoTrack
               trackRef={video}
               // There's no reason for this to be focusable
               tabIndex={-1}
               disablePictureInPicture
-              style={{ display: videoEnabled ? "block" : "none" }}
+              style={{ display: video && videoEnabled ? "block" : "none" }}
               data-testid="video"
             />
           )}
         </div>
         <div className={styles.fg}>
-          <div style={{ display: "flex", gap: "var(--cpd-space-1x)" }}>
+          <div className={styles.reactions}>
             <RaisedHandIndicator
               raisedHandTime={raisedHandTime}
               miniature={avatarSize < 96}
@@ -118,6 +120,11 @@ export const MediaView = forwardRef<HTMLDivElement, Props>(
               />
             )}
           </div>
+          {!video && !localParticipant && (
+            <div className={styles.status}>
+              {t("video_tile.waiting_for_media")}
+            </div>
+          )}
           {/* TODO: Bring this back once encryption status is less broken */}
           {/*encryptionStatus !== EncryptionStatus.Okay && (
             <div className={styles.status}>

src/tile/SpotlightTile.test.tsx

@@ -12,7 +12,11 @@ import userEvent from "@testing-library/user-event";
 import { of } from "rxjs";
 
 import { SpotlightTile } from "./SpotlightTile";
-import { withLocalMedia, withRemoteMedia } from "../utils/test";
+import {
+  mockRtcMembership,
+  withLocalMedia,
+  withRemoteMedia,
+} from "../utils/test";
 import { SpotlightTileViewModel } from "../state/TileViewModel";
 
 global.IntersectionObserver = class MockIntersectionObserver {
@@ -22,6 +26,7 @@ global.IntersectionObserver = class MockIntersectionObserver {
 
 test("SpotlightTile is accessible", async () => {
   await withRemoteMedia(
+    mockRtcMembership("@alice:example.org", "AAAA"),
     {
       rawDisplayName: "Alice",
       getMxcAvatarUrl: () => "mxc://adfsg",
@@ -29,6 +34,7 @@ test("SpotlightTile is accessible", async () => {
     {},
     async (vm1) => {
       await withLocalMedia(
+        mockRtcMembership("@bob:example.org", "BBBB"),
         {
           rawDisplayName: "Bob",
           getMxcAvatarUrl: () => "mxc://dlskf",

src/tile/SpotlightTile.tsx

@@ -49,12 +49,13 @@ interface SpotlightItemBaseProps {
   "data-id": string;
   targetWidth: number;
   targetHeight: number;
-  video: TrackReferenceOrPlaceholder;
+  video: TrackReferenceOrPlaceholder | undefined;
   member: RoomMember | undefined;
   unencryptedWarning: boolean;
   encryptionStatus: EncryptionStatus;
   displayName: string;
   "aria-hidden"?: boolean;
+  localParticipant: boolean;
 }
 
 interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps {
@@ -163,6 +164,7 @@ const SpotlightItem = forwardRef<HTMLDivElement, SpotlightItemProps>(
       displayName,
       encryptionStatus,
       "aria-hidden": ariaHidden,
+      localParticipant: vm.local,
     };
 
     return vm instanceof ScreenShareViewModel ? (
@@ -210,7 +212,9 @@ export const SpotlightTile = forwardRef<HTMLDivElement, Props>(
     const ref = useMergedRefs(ourRef, theirRef);
     const maximised = useObservableEagerState(vm.maximised);
     const media = useObservableEagerState(vm.media);
-    const [visibleId, setVisibleId] = useState(media[0].id);
+    const [visibleId, setVisibleId] = useState<string | undefined>(
+      media[0]?.id,
+    );
     const latestMedia = useLatest(media);
     const latestVisibleId = useLatest(visibleId);
     const visibleIndex = media.findIndex((vm) => vm.id === visibleId);

src/utils/test.ts

@@ -7,7 +7,20 @@ Please see LICENSE in the repository root for full details.
 import { map, Observable, of, SchedulerLike } from "rxjs";
 import { RunHelpers, TestScheduler } from "rxjs/testing";
 import { expect, vi } from "vitest";
-import { RoomMember, Room as MatrixRoom } from "matrix-js-sdk/src/matrix";
+import {
+  RoomMember,
+  Room as MatrixRoom,
+  MatrixEvent,
+  Room,
+  TypedEventEmitter,
+} from "matrix-js-sdk/src/matrix";
+import {
+  CallMembership,
+  Focus,
+  MatrixRTCSessionEvent,
+  MatrixRTCSessionEventHandlerMap,
+  SessionMembershipData,
+} from "matrix-js-sdk/src/matrixrtc";
 import {
   LocalParticipant,
   LocalTrackPublication,
@@ -100,11 +113,40 @@ function mockEmitter<T>(): EmitterMock<T> {
   };
 }
 
+export function mockRtcMembership(
+  user: string | RoomMember,
+  deviceId: string,
+  callId = "",
+  fociPreferred: Focus[] = [],
+  focusActive: Focus = { type: "oldest_membership" },
+  membership: Partial<SessionMembershipData> = {},
+): CallMembership {
+  const data: SessionMembershipData = {
+    application: "m.call",
+    call_id: callId,
+    device_id: deviceId,
+    foci_preferred: fociPreferred,
+    focus_active: focusActive,
+    ...membership,
+  };
+  const event = new MatrixEvent({
+    sender: typeof user === "string" ? user : user.userId,
+  });
+  return new CallMembership(event, data);
+}
+
 // Maybe it'd be good to move this to matrix-js-sdk? Our testing needs are
 // rather simple, but if one util to mock a member is good enough for us, maybe
 // it's useful for matrix-js-sdk consumers in general.
-export function mockMatrixRoomMember(member: Partial<RoomMember>): RoomMember {
+export function mockMatrixRoomMember(
-  return { ...mockEmitter(), ...member } as RoomMember;
+  rtcMembership: CallMembership,
+  member: Partial<RoomMember> = {},
+): RoomMember {
+  return {
+    ...mockEmitter(),
+    userId: rtcMembership.sender,
+    ...member,
+  } as RoomMember;
 }
 
 export function mockMatrixRoom(room: Partial<MatrixRoom>): MatrixRoom {
@@ -174,14 +216,15 @@ export function mockLocalParticipant(
 }
 
 export async function withLocalMedia(
-  member: Partial<RoomMember>,
+  localRtcMember: CallMembership,
+  roomMember: Partial<RoomMember>,
   continuation: (vm: LocalUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
   const localParticipant = mockLocalParticipant({});
   const vm = new LocalUserMediaViewModel(
     "local",
-    mockMatrixRoomMember(member),
+    mockMatrixRoomMember(localRtcMember, roomMember),
-    localParticipant,
+    of(localParticipant),
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
@@ -208,15 +251,16 @@ export function mockRemoteParticipant(
 }
 
 export async function withRemoteMedia(
-  member: Partial<RoomMember>,
+  localRtcMember: CallMembership,
+  roomMember: Partial<RoomMember>,
   participant: Partial<RemoteParticipant>,
   continuation: (vm: RemoteUserMediaViewModel) => void | Promise<void>,
 ): Promise<void> {
   const remoteParticipant = mockRemoteParticipant(participant);
   const vm = new RemoteUserMediaViewModel(
     "remote",
-    mockMatrixRoomMember(member),
+    mockMatrixRoomMember(localRtcMember, roomMember),
-    remoteParticipant,
+    of(remoteParticipant),
     {
       kind: E2eeType.PER_PARTICIPANT,
     },
@@ -244,3 +288,30 @@ export function mockMediaPlay(): string[] {
   };
   return audioIsPlaying;
 }
+
+export class MockRTCSession extends TypedEventEmitter<
+  MatrixRTCSessionEvent,
+  MatrixRTCSessionEventHandlerMap
+> {
+  public constructor(
+    public readonly room: Room,
+    private localMembership: CallMembership,
+    public memberships: CallMembership[] = [],
+  ) {
+    super();
+  }
+
+  public withMemberships(
+    rtcMembers: Observable<Partial<CallMembership>[]>,
+  ): MockRTCSession {
+    rtcMembers.subscribe((m) => {
+      const old = this.memberships;
+      // always prepend the local participant
+      const updated = [this.localMembership, ...(m as CallMembership[])];
+      this.memberships = updated;
+      this.emit(MatrixRTCSessionEvent.MembershipsChanged, old, updated);
+    });
+
+    return this;
+  }
+}