Merge pull request #3720 from element-hq/toger5/sdk-improvements-only

[MatrixRTC SDK] improvements - compatible with custom rtc application

.github/workflows/build-element-call.yaml

@@ -7,7 +7,7 @@ on:
         type: string
       package:
         type: string # This would ideally be a `choice` type, but that isn't supported yet
-        description: The package type to be built. Must be one of 'full' or 'embedded'
+        description: The package type to be built. Must be one of 'full', 'embedded', or 'sdk'
         required: true
       build_mode:
         type: string # This would ideally be a `choice` type, but that isn't supported yet

.github/workflows/build.yaml

@@ -69,3 +69,17 @@ jobs:
       SENTRY_URL: ${{ secrets.SENTRY_URL }}
       SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  build_sdk_element_call:
+    # Use the embedded package vite build
+    uses: ./.github/workflows/build-element-call.yaml
+    with:
+      package: sdk
+      vite_app_version: ${{ github.event.release.tag_name || github.sha }}
+      build_mode: ${{ github.event_name == 'pull_request' && contains(github.event.pull_request.labels.*.name, 'development build') && 'development' || 'production' }}
+    secrets:
+      SENTRY_ORG: ${{ secrets.SENTRY_ORG }}
+      SENTRY_PROJECT: ${{ secrets.SENTRY_PROJECT }}
+      SENTRY_URL: ${{ secrets.SENTRY_URL }}
+      SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/deploy-to-netlify.yaml

@@ -14,6 +14,10 @@ on:
       deployment_ref:
         required: true
         type: string
+      package:
+        required: true
+        type: string
+        description: Which package to deploy - 'full', 'embedded', or 'sdk'
       artifact_run_id:
         required: false
         type: string
@@ -50,7 +54,7 @@ jobs:
         with:
           github-token: ${{ secrets.ELEMENT_BOT_TOKEN }}
           run-id: ${{ inputs.artifact_run_id }}
-          name: build-output-full
+          name: build-output-${{ inputs.package }}
           path: webapp
 
       - name: Add redirects file
@@ -58,15 +62,17 @@ jobs:
         run: curl -s https://raw.githubusercontent.com/element-hq/element-call/main/config/netlify_redirects > webapp/_redirects
 
       - name: Add config file
-        run: curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview.json" > webapp/config.json
+        run: |
-
+          if [ "${{ inputs.package }}" = "full" ]; then
+            curl -s "https://raw.githubusercontent.com/${{ inputs.pr_head_full_name }}/${{ inputs.pr_head_ref }}/config/config_netlify_preview_sdk.json" > webapp/config.json
+          fi
       - name: ☁️ Deploy to Netlify
         id: netlify
         uses: nwtgck/actions-netlify@4cbaf4c08f1a7bfa537d6113472ef4424e4eb654 # v3.0
         with:
           publish-dir: webapp
           deploy-message: "Deploy from GitHub Actions"
-          alias: pr${{ inputs.pr_number }}
+          alias: ${{ inputs.package == 'sdk' && format('pr{0}-sdk', inputs.pr_number) || format('pr{0}', inputs.pr_number) }}
         env:
           NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
           NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}

.github/workflows/pr-deploy.yaml

@@ -20,7 +20,7 @@ jobs:
           owner: ${{ github.event.workflow_run.head_repository.owner.login }}
           branch: ${{ github.event.workflow_run.head_branch }}
 
-  netlify:
+  netlify-full:
     needs: prdetails
     permissions:
       deployments: write
@@ -31,6 +31,24 @@ jobs:
       pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
       pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
       deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
+      package: full
+    secrets:
+      ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
+      NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+      NETLIFY_SITE_ID: ${{ secrets.NETLIFY_SITE_ID }}
+
+  netlify-sdk:
+    needs: prdetails
+    permissions:
+      deployments: write
+    uses: ./.github/workflows/deploy-to-netlify.yaml
+    with:
+      artifact_run_id: ${{ github.event.workflow_run.id || github.run_id }}
+      pr_number: ${{ needs.prdetails.outputs.pr_number }}
+      pr_head_full_name: ${{ github.event.workflow_run.head_repository.full_name }}
+      pr_head_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.ref }}
+      deployment_ref: ${{ needs.prdetails.outputs.pr_data_json && fromJSON(needs.prdetails.outputs.pr_data_json).head.sha || github.ref || github.head_ref }}
+      package: sdk
     secrets:
       ELEMENT_BOT_TOKEN: ${{ secrets.ELEMENT_BOT_TOKEN }}
       NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}

package.json

@@ -13,8 +13,9 @@
     "build:embedded": "yarn build:full --config vite-embedded.config.js",
     "build:embedded:production": "yarn build:embedded",
     "build:embedded:development": "yarn build:embedded --mode development",
-    "build:sdk": "yarn build:full --config vite-sdk.config.js",
     "build:sdk:development": "yarn build:sdk --mode development",
+    "build:sdk": "yarn build:full --config vite-sdk.config.js",
+    "build:sdk:production": "yarn build:sdk",
     "serve": "vite preview",
     "prettier:check": "prettier -c .",
     "prettier:format": "prettier -w .",
@@ -104,7 +105,7 @@
     "livekit-client": "^2.13.0",
     "lodash-es": "^4.17.21",
     "loglevel": "^1.9.1",
-    "matrix-js-sdk": "matrix-org/matrix-js-sdk#develop",
+    "matrix-js-sdk": "matrix-org/matrix-js-sdk#6e3efef0c5f660df47cf00874927dec1c75cc3cf",
     "matrix-widget-api": "^1.16.1",
     "node-stdlib-browser": "^1.3.1",
     "normalize.css": "^8.0.1",

sdk/README.md

@@ -1,4 +1,4 @@
-# SDK mode
+# SDK mode (EXPERIMENTAL)
 
 EC can be build in sdk mode. This will result in a compiled js file that can be imported in very simple webapps.
 

sdk/helper.ts

@@ -12,15 +12,12 @@ Please see LICENSE in the repository root for full details.
 import { logger as rootLogger } from "matrix-js-sdk/lib/logger";
 import { scan } from "rxjs";
 
-import { widget as _widget } from "../src/widget";
+import { type WidgetHelpers } from "../src/widget";
 import { type LivekitRoomItem } from "../src/state/CallViewModel/CallViewModel";
 
 export const logger = rootLogger.getChild("[MatrixRTCSdk]");
 
-if (!_widget) throw Error("No widget. This webapp can only start as a widget");
+export const tryMakeSticky = (widget: WidgetHelpers): void => {
-export const widget = _widget;
-
-export const tryMakeSticky = (): void => {
   logger.info("try making sticky MatrixRTCSdk");
   void widget.api
     .setAlwaysOnScreen(true)

sdk/main.ts

@@ -6,6 +6,8 @@ Please see LICENSE in the repository root for full details.
 */
 
 /**
+ * EXPERIMENTAL
+ *
  * This file is the entrypoint for the sdk build of element call: `yarn build:sdk`
  * use in widgets.
  * It exposes the `createMatrixRTCSdk` which creates the `MatrixRTCSdk` interface (see below) that
@@ -30,8 +32,8 @@ import {
 } from "rxjs";
 import {
   type CallMembership,
-  MatrixRTCSession,
   MatrixRTCSessionEvent,
+  MatrixRTCSessionManager,
 } from "matrix-js-sdk/lib/matrixrtc";
 import {
   type Room as LivekitRoom,
@@ -50,14 +52,12 @@ import { getUrlParams } from "../src/UrlParams";
 import { MuteStates } from "../src/state/MuteStates";
 import { MediaDevices } from "../src/state/MediaDevices";
 import { E2eeType } from "../src/e2ee/e2eeType";
+import { currentAndPrev, logger, TEXT_LK_TOPIC, tryMakeSticky } from "./helper";
 import {
-  currentAndPrev,
+  ElementWidgetActions,
-  logger,
+  widget as _widget,
-  TEXT_LK_TOPIC,
+  initializeWidget,
-  tryMakeSticky,
+} from "../src/widget";
-  widget,
-} from "./helper";
-import { ElementWidgetActions, initializeWidget } from "../src/widget";
 import { type Connection } from "../src/state/CallViewModel/remoteMembers/Connection";
 
 interface MatrixRTCSdk {
@@ -68,7 +68,13 @@ interface MatrixRTCSdk {
   join: () => void;
   /** @throws on leave errors */
   leave: () => void;
-  data$: Observable<{ sender: string; data: string }>;
+  /**
+   * Ends the rtc sdk. This will unsubscribe any event listeners. And end the associated scope.
+   * No updates can be received from the rtc sdk. The sdk cannot be restarted after.
+   * A new sdk needs to be created via createMatrixRTCSdk.
+   */
+  stop: () => void;
+  data$: Observable<{ rtcBackendIdentity: string; data: string }>;
   /**
    * flattened list of members
    */
@@ -79,32 +85,54 @@ interface MatrixRTCSdk {
       participant: LocalParticipant | RemoteParticipant | null;
     }[]
   >;
+  /**
+   * flattened local members
+   */
+  localMember$: Behavior<{
+    connection: Connection | null;
+    membership: CallMembership;
+    participant: LocalParticipant | null;
+  } | null>;
   /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state  */
   connected$: Behavior<boolean>;
   sendData?: (data: unknown) => Promise<void>;
+  sendRoomMessage?: (message: string) => Promise<void>;
 }
 
 export async function createMatrixRTCSdk(
   application: string = "m.call",
   id: string = "",
+  sticky: boolean = false,
 ): Promise<MatrixRTCSdk> {
-  initializeWidget();
+  const scope = new ObservableScope();
+
+  // widget client
+  initializeWidget(application, true);
+  const widget = _widget;
+  if (!widget) throw Error("No widget. This webapp can only start as a widget");
   const client = await widget.client;
   logger.info("client created");
-  const scope = new ObservableScope();
+
+  // url params
   const { roomId } = getUrlParams();
   if (roomId === null) throw Error("could not get roomId from url params");
-
   const room = client.getRoom(roomId);
   if (room === null) throw Error("could not get room from client");
 
+  // rtc session
+  const slot = { application, id };
+  const rtcSessionManager = new MatrixRTCSessionManager(logger, client, slot);
+  rtcSessionManager.start();
+  const rtcSession = rtcSessionManager.getRoomSession(room);
+
+  // media devices
   const mediaDevices = new MediaDevices(scope);
   const muteStates = new MuteStates(scope, mediaDevices, {
-    audioEnabled: true,
+    audioEnabled: false,
-    videoEnabled: true,
+    videoEnabled: false,
   });
-  const slot = { application, id };
+
-  const rtcSession = new MatrixRTCSession(client, room, slot);
+  // call view model
   const callViewModel = createCallViewModel$(
     scope,
     rtcSession,
@@ -117,8 +145,9 @@ export async function createMatrixRTCSdk(
     constant({ supported: false, processor: undefined }),
   );
   logger.info("CallViewModelCreated");
+
   // create data listener
-  const data$ = new Subject<{ sender: string; data: string }>();
+  const data$ = new Subject<{ rtcBackendIdentity: string; data: string }>();
 
   const lkTextStreamHandlerFunction = async (
     reader: TextStreamReader,
@@ -140,7 +169,7 @@ export async function createMatrixRTCSdk(
     if (participants && participants.includes(participantInfo.identity)) {
       const text = await reader.readAll();
       logger.info(`Received text: ${text}`);
-      data$.next({ sender: participantInfo.identity, data: text });
+      data$.next({ rtcBackendIdentity: participantInfo.identity, data: text });
     } else {
       logger.warn(
         "Received text from unknown participant",
@@ -230,6 +259,16 @@ export async function createMatrixRTCSdk(
     }
   };
 
+  const sendRoomMessage = async (message: string): Promise<void> => {
+    const messageString = JSON.stringify(message);
+    logger.info("try sending to room: ", messageString);
+    try {
+      await client.sendTextMessage(room.roomId, message);
+    } catch (e) {
+      logger.error("failed sending to room: ", messageString, e);
+    }
+  };
+
   // after hangup gets called
   const leaveSubs = callViewModel.leave$.subscribe(() => {
     const scheduleWidgetCloseOnLeave = async (): Promise<void> => {
@@ -257,9 +296,6 @@ export async function createMatrixRTCSdk(
 
     // schedule close first and then leave (scope.end)
     void scheduleWidgetCloseOnLeave();
-
-    // actual hangup (ending scope will send the leave event.. its kinda odd. since you might end up closing the widget too fast)
-    scope.end();
   });
 
   logger.info("createMatrixRTCSdk done");
@@ -267,15 +303,40 @@ export async function createMatrixRTCSdk(
   return {
     join: (): void => {
       // first lets try making the widget sticky
-      tryMakeSticky();
+      if (sticky) tryMakeSticky(widget);
       callViewModel.join();
     },
     leave: (): void => {
-      callViewModel.hangup();
+      callViewModel.leave();
+    },
+    stop: (): void => {
       leaveSubs.unsubscribe();
       livekitRoomItemsSub.unsubscribe();
+      scope.end();
     },
     data$,
+    localMember$: scope.behavior(
+      callViewModel.localMatrixLivekitMember$.pipe(
+        tap((member) =>
+          logger.info("localMatrixLivekitMember$ next: ", member),
+        ),
+        switchMap((member) => {
+          if (member === null) return of(null);
+          return combineLatest([
+            member.connection$,
+            member.membership$,
+            member.participant.value$,
+          ]).pipe(
+            map(([connection, membership, participant]) => ({
+              connection,
+              membership,
+              participant,
+            })),
+          );
+        }),
+        tap((member) => logger.info("localMember$ next: ", member)),
+      ),
+    ),
     connected$: callViewModel.connected$,
     members$: scope.behavior(
       callViewModel.matrixLivekitMembers$.pipe(
@@ -302,5 +363,6 @@ export async function createMatrixRTCSdk(
       [],
     ),
     sendData,
+    sendRoomMessage,
   };
 }

src/state/CallViewModel/CallViewModel.ts

@@ -217,15 +217,23 @@ export interface CallViewModel {
     "unknown" | "ringing" | "timeout" | "decline" | "success" | null
   >;
   /** Observable that emits when the user should leave the call (hangup pressed, widget action, error).
-   * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is by ending the scope.
+   * THIS DOES NOT LEAVE THE CALL YET. The only way to leave the call (send the hangup event) is
+   *  - by ending the scope
+   *  - or calling requestDisconnect
+   *
+   * TODO: it seems more reasonable to add a leave() method (that calls requestDisconnect) that will then update leave$ and remove the hangup pattern
    */
   leave$: Observable<"user" | AutoLeaveReason>;
-  /** Call to initiate hangup. Use in conbination with reconnectino state track the async hangup process. */
+  /** Call to initiate hangup. Use in conbination with reconnection state track the async hangup process. */
   hangup: () => void;
 
   // joining
   join: () => void;
 
+  /**
+   * calls requestDisconnect. The async leave state can than be observed via connected$
+   */
+  leave: () => void;
   // screen sharing
   /**
    * Callback to toggle screen sharing. If null, screen sharing is not possible.
@@ -1497,6 +1505,7 @@ export function createCallViewModel$(
     leave$: leave$,
     hangup: (): void => userHangup$.next(),
     join: localMembership.requestJoinAndPublish,
+    leave: localMembership.requestDisconnect,
     toggleScreenSharing: toggleScreenSharing,
     sharingScreen$: sharingScreen$,
 

src/widget.test.ts

@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
 Please see LICENSE in the repository root for full details.
 */
 
-import { beforeAll, describe, expect, vi, it } from "vitest";
+import { describe, expect, vi, it, beforeEach } from "vitest";
 import { createRoomWidgetClient, EventType } from "matrix-js-sdk";
 
 import { getUrlParams } from "./UrlParams";
@@ -35,11 +35,14 @@ vi.mock("./UrlParams", () => ({
   })),
 }));
 
-initializeWidget();
+beforeEach(() => {
-describe("widget", () => {
+  createRoomWidgetClientSpy.mockClear();
-  beforeAll(() => {});
+});
 
+describe("widget", () => {
   it("should create an embedded client with the correct params", () => {
+    initializeWidget("ANYRTCAPP");
+
     expect(getUrlParams()).toStrictEqual({
       widgetId: "id",
       parentUrl: "http://parentUrl",
@@ -66,13 +69,16 @@ describe("widget", () => {
     ];
 
     const sendState = [
-      "myYser", // Legacy call membership events
+      { eventType: "org.matrix.msc3401.call.member", stateKey: "myYser" }, // Legacy call membership events
-      `_myYser_AAAAA_m.call`, // Session membership events
+      {
-      `myYser_AAAAA_m.call`, // The above with no leading underscore, for room versions whose auth rules allow it
+        eventType: "org.matrix.msc3401.call.member",
-    ].map((stateKey) => ({
+        stateKey: `_myYser_AAAAA_ANYRTCAPP`,
-      eventType: EventType.GroupCallMemberPrefix,
+      }, // Session membership events
-      stateKey,
+      {
-    }));
+        eventType: "org.matrix.msc3401.call.member",
+        stateKey: `myYser_AAAAA_ANYRTCAPP`,
+      }, // The above with no leading underscore, for room versions whose auth rules allow it
+    ];
     const receiveState = [
       { eventType: EventType.RoomCreate },
       { eventType: EventType.RoomName },
@@ -124,4 +130,32 @@ describe("widget", () => {
     });
     expect(createRoomWidgetClientSpy.mock.calls[0][4]).toStrictEqual(false);
   });
+
+  it("should request send message permission if requested", () => {
+    initializeWidget("ANYRTCAPP", true);
+    expect(createRoomWidgetClientSpy).toHaveBeenLastCalledWith(
+      expect.anything(),
+      // capabilities
+      expect.objectContaining({
+        sendEvent: expect.arrayContaining(["m.room.message"]),
+      }),
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+    );
+  });
+
+  it("should not request send message permission when not requested", () => {
+    initializeWidget("", false);
+    expect(createRoomWidgetClientSpy).toHaveBeenLastCalledWith(
+      expect.anything(),
+      // capabilities
+      expect.objectContaining({
+        sendEvent: expect.not.arrayContaining(["m.room.message"]),
+      }),
+      expect.anything(),
+      expect.anything(),
+      expect.anything(),
+    );
+  });
 });

src/widget.ts

@@ -68,7 +68,10 @@ export let widget: WidgetHelpers | null;
  */
 // this needs to be a seperate call and cannot be done on import to allow us to spy on methods in here before
 // execution.
-export const initializeWidget = (): void => {
+export const initializeWidget = (
+  rtcApplication: string = "m.call",
+  sendRoomEvents = false,
+): void => {
   try {
     const {
       widgetId,
@@ -116,6 +119,9 @@ export const initializeWidget = (): void => {
         EventType.CallNotify, // Sent as a deprecated fallback
         EventType.RTCNotification,
       ];
+      if (sendRoomEvents) {
+        sendEvent.push(EventType.RoomMessage);
+      }
       const sendRecvEvent = [
         "org.matrix.rageshake_request",
         EventType.CallEncryptionKeysPrefix,
@@ -128,8 +134,8 @@ export const initializeWidget = (): void => {
 
       const sendState = [
         userId, // Legacy call membership events
-        `_${userId}_${deviceId}_m.call`, // Session membership events
+        `_${userId}_${deviceId}_${rtcApplication}`, // Session membership events
-        `${userId}_${deviceId}_m.call`, // The above with no leading underscore, for room versions whose auth rules allow it
+        `${userId}_${deviceId}_${rtcApplication}`, // The above with no leading underscore, for room versions whose auth rules allow it
       ].map((stateKey) => ({
         eventType: EventType.GroupCallMemberPrefix,
         stateKey,

yarn.lock

@@ -8364,7 +8364,7 @@ __metadata:
     livekit-client: "npm:^2.13.0"
     lodash-es: "npm:^4.17.21"
     loglevel: "npm:^1.9.1"
-    matrix-js-sdk: "matrix-org/matrix-js-sdk#develop"
+    matrix-js-sdk: "matrix-org/matrix-js-sdk#6e3efef0c5f660df47cf00874927dec1c75cc3cf"
     matrix-widget-api: "npm:^1.16.1"
     node-stdlib-browser: "npm:^1.3.1"
     normalize.css: "npm:^8.0.1"
@@ -11452,9 +11452,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"matrix-js-sdk@matrix-org/matrix-js-sdk#develop":
+"matrix-js-sdk@matrix-org/matrix-js-sdk#6e3efef0c5f660df47cf00874927dec1c75cc3cf":
   version: 40.1.0
-  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=f2157f28bbadf2898fe21991f69ccb2af40df326"
+  resolution: "matrix-js-sdk@https://github.com/matrix-org/matrix-js-sdk.git#commit=6e3efef0c5f660df47cf00874927dec1c75cc3cf"
   dependencies:
     "@babel/runtime": "npm:^7.12.5"
     "@matrix-org/matrix-sdk-crypto-wasm": "npm:^17.0.0"
@@ -11470,7 +11470,7 @@ __metadata:
     sdp-transform: "npm:^3.0.0"
     unhomoglyph: "npm:^1.0.6"
     uuid: "npm:13"
-  checksum: 10c0/d646b9214abbf0b9126760105edd9c57be7ffe8b53ae4acd5fefe841a51ad7d78fa57130922b3eac65ff2266b43f31ea60b4bdda9481e6bf8f1808d96726ed8a
+  checksum: 10c0/2c4db56fd0164d801c2f125ab2a442e3659314d4cc2fd640ea152b829d0db8b05ff808020e387a761afde4ff7a07b271c25431337de9f7c765c523c8cd837e36
   languageName: node
   linkType: hard