(EXPERIMENTAL) Prototyping for reliable call membership and presence

.github/workflows/element-call.yaml

@@ -26,6 +26,7 @@ jobs:
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
         with:
           cache: "yarn"
+          node-version-file: ".nvmrc"
       - name: Install dependencies
         run: "yarn install"
       - name: Build

.github/workflows/lint.yaml

@@ -12,6 +12,7 @@ jobs:
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
         with:
           cache: "yarn"
+          node-version-file: ".nvmrc"
       - name: Install dependencies
         run: "yarn install"
       - name: Prettier

.github/workflows/test.yaml

@@ -14,6 +14,7 @@ jobs:
         uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
         with:
           cache: "yarn"
+          node-version-file: ".nvmrc"
       - name: Install dependencies
         run: "yarn install"
       - name: Vitest

.github/workflows/translations-download.yaml

@@ -18,6 +18,7 @@ jobs:
       - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4
         with:
           cache: "yarn"
+          node-version-file: ".nvmrc"
 
       - name: Install Deps
         run: "yarn install --frozen-lockfile"

.nvmrc

@@ -0,0 +1 @@
+20

package.json

@@ -23,6 +23,7 @@
     "@livekit/components-core": "^0.10.0",
     "@livekit/components-react": "^2.0.0",
     "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz",
+    "@nkzw/use-relative-time": "^1.1.0",
     "@opentelemetry/api": "^1.4.0",
     "@opentelemetry/context-zone": "^1.9.1",
     "@opentelemetry/exporter-jaeger": "^1.9.1",
@@ -63,7 +64,7 @@
     "i18next-http-backend": "^2.0.0",
     "livekit-client": "^2.0.2",
     "lodash": "^4.17.21",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#238eea0ef5c82d0a11b8d5cc5c04104d6c94c4c1",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c795c7eb656f2f6136ac4c6b17015c0f000df9a6",
     "matrix-widget-api": "^1.3.1",
     "normalize.css": "^8.0.1",
     "pako": "^2.0.4",

src/room/InCallView.tsx

@@ -238,7 +238,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
     const noControls = reducedControls && bounds.height <= 400;
 
     const vm = useCallViewModel(
-      rtcSession.room,
+      rtcSession,
       livekitRoom,
       matrixInfo.e2eeSystem.kind !== E2eeType.NONE,
       connState,

src/state/CallViewModel.ts

@@ -52,6 +52,8 @@ import {
 } from "rxjs";
 import { StateObservable, state } from "@react-rxjs/core";
 import { logger } from "matrix-js-sdk/src/logger";
+import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
+import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership";
 
 import { ViewModel } from "./ViewModel";
 import { useObservable } from "./useObservable";
@@ -64,6 +66,7 @@ import {
   MediaViewModel,
   UserMediaViewModel,
   ScreenShareViewModel,
+  MembershipOnlyViewModel,
 } from "./MediaViewModel";
 import { finalizeValue } from "../observable-utils";
 import { ObservableScope } from "./ObservableScope";
@@ -204,25 +207,45 @@ class ScreenShare {
   }
 }
 
-type MediaItem = UserMedia | ScreenShare;
+class MembershipOnly {
+  public readonly vm: MembershipOnlyViewModel;
 
-function findMatrixMember(
-  room: MatrixRoom,
-  id: string,
-): RoomMember | undefined {
-  if (!id) return undefined;
+  public constructor(member: RoomMember) {
+    this.vm = new MembershipOnlyViewModel(member);
+  }
+
+  public destroy(): void {
+    this.vm.destroy();
+  }
+}
+
+type MediaItem = UserMedia | ScreenShare | MembershipOnly;
 
+function matrixUserIdFromParticipantId(id: string): string | undefined {
+  if (!id) return undefined;
   const parts = id.split(":");
   // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
   if (parts.length < 3) {
     logger.warn(
-      "Livekit participants ID doesn't look like a userId:deviceId combination",
+      `Livekit participants ID doesn't look like a userId:deviceId combination: ${id}`,
     );
     return undefined;
   }
 
   parts.pop();
   const userId = parts.join(":");
+  return userId;
+}
+
+function findMatrixMember(
+  room: MatrixRoom,
+  id: string,
+): RoomMember | undefined {
+  const userId = matrixUserIdFromParticipantId(id);
+
+  if (!userId) {
+    return undefined;
+  }
 
   return room.getMember(userId) ?? undefined;
 }
@@ -304,6 +327,24 @@ export class CallViewModel extends ViewModel {
       },
     );
 
+  private readonly membershipsWithoutParticipant = combineLatest([
+    of(this.rtcSession.memberships),
+    this.remoteParticipants,
+    of(this.livekitRoom.localParticipant),
+  ]).pipe(
+    scan((prev, [memberships, remoteParticipants, localParticipant]) => {
+      const participantIds = new Set(
+        remoteParticipants.map((p) =>
+          matrixUserIdFromParticipantId(p.identity),
+        ),
+      );
+      participantIds.add(
+        matrixUserIdFromParticipantId(localParticipant.identity),
+      );
+      return memberships.filter((m) => !participantIds.has(m.sender ?? ""));
+    }, [] as CallMembership[]),
+  );
+
   private readonly mediaItems: StateObservable<MediaItem[]> = state(
     combineLatest([
       this.remoteParticipants,
@@ -495,90 +536,136 @@ export class CallViewModel extends ViewModel {
       combineLatest([
         this.remoteParticipants,
         observeParticipantMedia(this.livekitRoom.localParticipant),
+        this.membershipsWithoutParticipant,
       ]).pipe(
-        scan((ts, [remoteParticipants, { participant: localParticipant }]) => {
-          const ps = [localParticipant, ...remoteParticipants];
-          const tilesById = new Map(ts.map((t) => [t.id, t]));
-          const now = Date.now();
-          let allGhosts = true;
-
-          const newTiles = ps.flatMap((p) => {
-            const userMediaId = p.identity;
-            const member = findMatrixMember(this.matrixRoom, userMediaId);
-            allGhosts &&= member === undefined;
-            const spokeRecently =
-              p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
-
-            // We always start with a local participant with the empty string as
-            // their ID before we're connected, this is fine and we'll be in
-            // "all ghosts" mode.
-            if (userMediaId !== "" && member === undefined) {
-              logger.warn(
-                `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
-              );
-            }
+        scan(
+          (
+            ts,
+            [
+              remoteParticipants,
+              { participant: localParticipant },
+              membershipsWithoutParticipant,
+            ],
+          ) => {
+            const ps = [
+              localParticipant,
+              ...remoteParticipants,
+              ...membershipsWithoutParticipant,
+            ];
+            const tilesById = new Map(ts.map((t) => [t.id, t]));
+            const now = Date.now();
+            let allGhosts = true;
+
+            const newTiles = ps.flatMap((p) => {
+              if (p instanceof CallMembership) {
+                const userId = p.sender ?? "";
+                const member = this.matrixRoom.getMember(userId);
+                if (!member) {
+                  logger.warn(
+                    `Ruh, roh! No matrix member found for call membership '${userId}': ignoring`,
+                  );
+                  return [];
+                }
+                const membershipOnlyVm =
+                  tilesById.get(userId)?.data ??
+                  new MembershipOnlyViewModel(member);
+                tilesById.delete(userId);
+
+                const membershipOnlyTile: TileDescriptor<MediaViewModel> = {
+                  id: userId,
+                  focused: false,
+                  isPresenter: false,
+                  isSpeaker: false,
+                  hasVideo: false,
+                  local: false,
+                  largeBaseSize: false,
+                  data: membershipOnlyVm,
+                };
+                return [membershipOnlyTile];
+              }
 
-            const userMediaVm =
-              tilesById.get(userMediaId)?.data ??
-              new UserMediaViewModel(userMediaId, member, p, this.encrypted);
-            tilesById.delete(userMediaId);
-
-            const userMediaTile: TileDescriptor<MediaViewModel> = {
-              id: userMediaId,
-              focused: false,
-              isPresenter: p.isScreenShareEnabled,
-              isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
-              hasVideo: p.isCameraEnabled,
-              local: p.isLocal,
-              largeBaseSize: false,
-              data: userMediaVm,
-            };
-
-            if (p.isScreenShareEnabled) {
-              const screenShareId = `${userMediaId}:screen-share`;
-              const screenShareVm =
-                tilesById.get(screenShareId)?.data ??
-                new ScreenShareViewModel(
-                  screenShareId,
-                  member,
-                  p,
-                  this.encrypted,
+              const userMediaId = p.identity;
+              const member = findMatrixMember(this.matrixRoom, userMediaId);
+              allGhosts &&= member === undefined;
+              const spokeRecently =
+                p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
+
+              // We always start with a local participant with the empty string as
+              // their ID before we're connected, this is fine and we'll be in
+              // "all ghosts" mode.
+              if (userMediaId !== "" && member === undefined) {
+                logger.warn(
+                  `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
                 );
-              tilesById.delete(screenShareId);
-
-              const screenShareTile: TileDescriptor<MediaViewModel> = {
-                id: screenShareId,
-                focused: true,
-                isPresenter: false,
-                isSpeaker: false,
-                hasVideo: true,
+              }
+
+              const userMediaVm =
+                tilesById.get(userMediaId)?.data ??
+                new UserMediaViewModel(userMediaId, member, p, this.encrypted);
+              tilesById.delete(userMediaId);
+
+              const userMediaTile: TileDescriptor<MediaViewModel> = {
+                id: userMediaId,
+                focused: false,
+                isPresenter: p.isScreenShareEnabled,
+                isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
+                hasVideo: p.isCameraEnabled,
                 local: p.isLocal,
-                largeBaseSize: true,
-                placeNear: userMediaId,
-                data: screenShareVm,
+                largeBaseSize: false,
+                data: userMediaVm,
               };
-              return [userMediaTile, screenShareTile];
-            } else {
-              return [userMediaTile];
-            }
-          });
 
-          // Any tiles left in the map are unused and should be destroyed
-          for (const t of tilesById.values()) t.data.destroy();
+              if (p.isScreenShareEnabled) {
+                const screenShareId = `${userMediaId}:screen-share`;
+                const screenShareVm =
+                  tilesById.get(screenShareId)?.data ??
+                  new ScreenShareViewModel(
+                    screenShareId,
+                    member,
+                    p,
+                    this.encrypted,
+                  );
+                tilesById.delete(screenShareId);
+
+                const screenShareTile: TileDescriptor<MediaViewModel> = {
+                  id: screenShareId,
+                  focused: true,
+                  isPresenter: false,
+                  isSpeaker: false,
+                  hasVideo: true,
+                  local: p.isLocal,
+                  largeBaseSize: true,
+                  placeNear: userMediaId,
+                  data: screenShareVm,
+                };
+                return [userMediaTile, screenShareTile];
+              } else {
+                return [userMediaTile];
+              }
+            });
 
-          // If every item is a ghost, that probably means we're still connecting
-          // and shouldn't bother showing anything yet
-          return allGhosts ? [] : newTiles;
-        }, [] as TileDescriptor<MediaViewModel>[]),
+            // Any tiles left in the map are unused and should be destroyed
+            for (const t of tilesById.values()) t.data.destroy();
+
+            // If every item is a ghost, that probably means we're still connecting
+            // and shouldn't bother showing anything yet
+            return allGhosts ? [] : newTiles;
+          },
+          [] as TileDescriptor<MediaViewModel>[],
+        ),
         finalizeValue((ts) => {
           for (const t of ts) t.data.destroy();
         }),
       ),
     );
 
+  private get matrixRoom(): MatrixRoom {
+    return this.rtcSession.room;
+  }
+
   public constructor(
     // A call is permanently tied to a single Matrix room and LiveKit room
-    private readonly matrixRoom: MatrixRoom,
+    private readonly rtcSession: MatrixRTCSession,
     private readonly livekitRoom: LivekitRoom,
     private readonly encrypted: boolean,
     private readonly connectionState: Observable<ECConnectionState>,
@@ -588,25 +675,25 @@ export class CallViewModel extends ViewModel {
 }
 
 export function useCallViewModel(
-  matrixRoom: MatrixRoom,
+  rtcSession: MatrixRTCSession,
   livekitRoom: LivekitRoom,
   encrypted: boolean,
   connectionState: ECConnectionState,
 ): CallViewModel {
-  const prevMatrixRoom = usePrevious(matrixRoom);
+  const prevRTCSession = usePrevious(rtcSession);
   const prevLivekitRoom = usePrevious(livekitRoom);
   const prevEncrypted = usePrevious(encrypted);
   const connectionStateObservable = useObservable(connectionState);
 
   const vm = useRef<CallViewModel>();
   if (
-    matrixRoom !== prevMatrixRoom ||
+    rtcSession !== prevRTCSession ||
     livekitRoom !== prevLivekitRoom ||
     encrypted !== prevEncrypted
   ) {
     vm.current?.destroy();
     vm.current = new CallViewModel(
-      matrixRoom,
+      rtcSession,
       livekitRoom,
       encrypted,
       connectionStateObservable,