Download avatars using the Widget API

If we can't authenticate media because we're
running as a widget, use the MC4039 widget API
instead of a direct fetch to download the avatar.

src/Avatar.test.tsx

@@ -13,6 +13,8 @@ import { type FC, type PropsWithChildren } from "react";
 import { ClientContextProvider } from "./ClientContext";
 import { Avatar } from "./Avatar";
 import { mockMatrixRoomMember, mockRtcMembership } from "./utils/test";
+import EventEmitter from "events";
+import { widget } from "./widget";
 
 const TestComponent: FC<
   PropsWithChildren<{ client: MatrixClient; supportsThumbnails?: boolean }>
@@ -40,6 +42,12 @@ const TestComponent: FC<
   );
 };
 
+vi.mock("./widget", () => ({
+  widget: {
+    api: { downloadFile: vi.fn() },
+  },
+}));
+
 afterEach(() => {
   vi.unstubAllGlobals();
 });
@@ -154,3 +162,46 @@ test("should attempt to fetch authenticated media", async () => {
     headers: { Authorization: `Bearer ${accessToken}` },
   });
 });
+
+test("should use widget API when unable to authenticate media", async () => {
+  const expectedMXCUrl = "mxc://example.org/alice-avatar";
+  const expectedObjectURL = "my-object-url";
+  const theBlob = new Blob([]);
+
+  // vitest doesn't have a implementation of create/revokeObjectURL, so we need
+  // to delete the property. It's a bit odd, but it works.
+  Reflect.deleteProperty(global.window.URL, "createObjectURL");
+  globalThis.URL.createObjectURL = vi.fn().mockReturnValue(expectedObjectURL);
+  Reflect.deleteProperty(global.window.URL, "revokeObjectURL");
+  globalThis.URL.revokeObjectURL = vi.fn();
+
+  const client = vi.mocked<MatrixClient>({
+    getAccessToken: () => undefined,
+  } as unknown as MatrixClient);
+
+  vi.spyOn(widget!.api, "downloadFile").mockResolvedValue({ file: theBlob });
+  const member = mockMatrixRoomMember(
+    mockRtcMembership("@alice:example.org", "AAAA"),
+    {
+      getMxcAvatarUrl: () => expectedMXCUrl,
+    },
+  );
+  const displayName = "Alice";
+  render(
+    <TestComponent client={client}>
+      <Avatar
+        id={member.userId}
+        name={displayName}
+        size={96}
+        src={member.getMxcAvatarUrl()}
+      />
+    </TestComponent>,
+  );
+
+  // Fetch is asynchronous, so wait for this to resolve.
+  await vi.waitUntil(() =>
+    document.querySelector(`img[src='${expectedObjectURL}']`),
+  );
+
+  expect(widget!.api.downloadFile).toBeCalledWith(expectedMXCUrl);
+});

src/Avatar.tsx

@@ -15,7 +15,9 @@ import {
 import { Avatar as CompoundAvatar } from "@vector-im/compound-web";
 import { type MatrixClient } from "matrix-js-sdk";
 
-import { useClientState } from "./ClientContext";
+import { ClientState, useClientState } from "./ClientContext";
+import { widget } from "./widget";
+import { WidgetApi } from "matrix-widget-api";
 
 export enum Size {
   XS = "xs",
@@ -86,33 +88,17 @@ export const Avatar: FC<Props> = ({
   const [avatarUrl, setAvatarUrl] = useState<string | undefined>(undefined);
 
   useEffect(() => {
-    if (clientState?.state !== "valid") {
+    if (!src) {
-      return;
-    }
-    const { authenticated, supportedFeatures } = clientState;
-    const client = authenticated?.client;
-
-    if (!client || !src || !sizePx || !supportedFeatures.thumbnails) {
-      return;
-    }
-
-    const token = client.getAccessToken();
-    if (!token) {
-      return;
-    }
-    const resolveSrc = getAvatarUrl(client, src, sizePx);
-    if (!resolveSrc) {
       setAvatarUrl(undefined);
       return;
     }
 
     let objectUrl: string | undefined;
-    fetch(resolveSrc, {
+
-      headers: {
+    getAvatarFromServer(clientState, src, sizePx) // Try to download directly from the mxc://
-        Authorization: `Bearer ${token}`,
+      .catch((ex) => {
-      },
+        return getAvatarFromWidget(widget?.api, src); // Fallback to trying to use the MSC4039 Widget API
-    })
+      })
-      .then(async (req) => req.blob())
       .then((blob) => {
         objectUrl = URL.createObjectURL(blob);
         setAvatarUrl(objectUrl);
@@ -140,3 +126,64 @@ export const Avatar: FC<Props> = ({
     />
   );
 };
+
+async function getAvatarFromServer(
+  clientState: ClientState | undefined,
+  src: string,
+  sizePx: number | undefined,
+): Promise<Blob> {
+  if (clientState?.state !== "valid") {
+    throw new Error("Client state must be valid");
+  }
+  if (!sizePx) {
+    throw new Error("size must be supplied");
+  }
+
+  const { authenticated, supportedFeatures } = clientState;
+  const client = authenticated?.client;
+  if (!client) {
+    throw new Error("Client must be supplied");
+  }
+  if (!supportedFeatures.thumbnails) {
+    throw new Error("Thumbnails are not supported");
+  }
+
+  const httpSrc = getAvatarUrl(client, src, sizePx);
+  if (!httpSrc) {
+    throw new Error("Failed to get http avatar URL");
+  }
+
+  const token = client.getAccessToken();
+  if (!token) {
+    throw new Error("Failed to get access token");
+  }
+
+  const request = await fetch(httpSrc, {
+    headers: {
+      Authorization: `Bearer ${token}`,
+    },
+  });
+
+  const blob = await request.blob();
+
+  return blob;
+}
+
+async function getAvatarFromWidget(
+  api: WidgetApi | undefined,
+  src: string,
+): Promise<Blob> {
+  if (!api) {
+    throw new Error("No widget api given");
+  }
+
+  const response = await api.downloadFile(src);
+  const file = response.file;
+
+  // element-web sends a Blob, and the MSC4039 is considering changing the spec to strictly Blob, so only handling that
+  if (!(file instanceof Blob)) {
+    throw new Error("Downloaded file is not a Blob");
+  }
+
+  return file;
+}

src/widget.ts

@@ -93,6 +93,7 @@ export const initializeWidget = (
       logger.info("Widget API is available");
       const api = new WidgetApi(widgetId, parentOrigin);
       api.requestCapability(MatrixCapabilities.AlwaysOnScreen);
+      api.requestCapability(MatrixCapabilities.MSC4039DownloadFile);
 
       // Set up the lazy action emitter, but only for select actions that we
       // intend for the app to handle