From 76a494464e0d110f56fa7fd3640497a19c7d56f8 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 17 Jul 2024 11:08:46 +0100 Subject: [PATCH 1/4] Prototyping of showing video tile for every call membership ...even if they don't have a LK participant --- package.json | 1 + src/room/InCallView.tsx | 2 +- src/state/CallViewModel.ts | 247 +++++++++++++++++++++++------------ src/state/MediaViewModel.ts | 17 ++- src/video-grid/VideoTile.tsx | 108 ++++++++++++++- yarn.lock | 5 + 6 files changed, 294 insertions(+), 86 deletions(-) diff --git a/package.json b/package.json index ae7a259c7..1780075fb 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "@livekit/components-core": "^0.10.0", "@livekit/components-react": "^2.0.0", "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz", + "@nkzw/use-relative-time": "^1.1.0", "@opentelemetry/api": "^1.4.0", "@opentelemetry/context-zone": "^1.9.1", "@opentelemetry/exporter-jaeger": "^1.9.1", diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 94edace7d..77151be0e 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -238,7 +238,7 @@ export const InCallView: FC = subscribe( const noControls = reducedControls && bounds.height <= 400; const vm = useCallViewModel( - rtcSession.room, + rtcSession, livekitRoom, matrixInfo.e2eeSystem.kind !== E2eeType.NONE, connState, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4ad2f0242..1bd7613ad 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -52,6 +52,8 @@ import { } from "rxjs"; import { StateObservable, state } from "@react-rxjs/core"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; import { ViewModel } from "./ViewModel"; import { useObservable } from "./useObservable"; @@ -64,6 +66,7 @@ import { MediaViewModel, UserMediaViewModel, ScreenShareViewModel, + MembershipOnlyViewModel, } from "./MediaViewModel"; import { finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; @@ -204,25 +207,45 @@ class ScreenShare { } } -type MediaItem = UserMedia | ScreenShare; +class MembershipOnly { + public readonly vm: MembershipOnlyViewModel; -function findMatrixMember( - room: MatrixRoom, - id: string, -): RoomMember | undefined { - if (!id) return undefined; + public constructor(member: RoomMember) { + this.vm = new MembershipOnlyViewModel(member); + } + + public destroy(): void { + this.vm.destroy(); + } +} + +type MediaItem = UserMedia | ScreenShare | MembershipOnly; +function matrixUserIdFromParticipantId(id: string): string | undefined { + if (!id) return undefined; const parts = id.split(":"); // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon if (parts.length < 3) { logger.warn( - "Livekit participants ID doesn't look like a userId:deviceId combination", + `Livekit participants ID doesn't look like a userId:deviceId combination: ${id}`, ); return undefined; } parts.pop(); const userId = parts.join(":"); + return userId; +} + +function findMatrixMember( + room: MatrixRoom, + id: string, +): RoomMember | undefined { + const userId = matrixUserIdFromParticipantId(id); + + if (!userId) { + return undefined; + } return room.getMember(userId) ?? undefined; } @@ -304,6 +327,24 @@ export class CallViewModel extends ViewModel { }, ); + private readonly membershipsWithoutParticipant = combineLatest([ + of(this.rtcSession.memberships), + this.remoteParticipants, + of(this.livekitRoom.localParticipant), + ]).pipe( + scan((prev, [memberships, remoteParticipants, localParticipant]) => { + const participantIds = new Set( + remoteParticipants.map((p) => + matrixUserIdFromParticipantId(p.identity), + ), + ); + participantIds.add( + matrixUserIdFromParticipantId(localParticipant.identity), + ); + return memberships.filter((m) => !participantIds.has(m.sender ?? "")); + }, [] as CallMembership[]), + ); + private readonly mediaItems: StateObservable = state( combineLatest([ this.remoteParticipants, @@ -495,90 +536,136 @@ export class CallViewModel extends ViewModel { combineLatest([ this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), + this.membershipsWithoutParticipant, ]).pipe( - scan((ts, [remoteParticipants, { participant: localParticipant }]) => { - const ps = [localParticipant, ...remoteParticipants]; - const tilesById = new Map(ts.map((t) => [t.id, t])); - const now = Date.now(); - let allGhosts = true; - - const newTiles = ps.flatMap((p) => { - const userMediaId = p.identity; - const member = findMatrixMember(this.matrixRoom, userMediaId); - allGhosts &&= member === undefined; - const spokeRecently = - p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; - - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (userMediaId !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, - ); - } + scan( + ( + ts, + [ + remoteParticipants, + { participant: localParticipant }, + membershipsWithoutParticipant, + ], + ) => { + const ps = [ + localParticipant, + ...remoteParticipants, + ...membershipsWithoutParticipant, + ]; + const tilesById = new Map(ts.map((t) => [t.id, t])); + const now = Date.now(); + let allGhosts = true; + + const newTiles = ps.flatMap((p) => { + if (p instanceof CallMembership) { + const userId = p.sender ?? ""; + const member = this.matrixRoom.getMember(userId); + if (!member) { + logger.warn( + `Ruh, roh! No matrix member found for call membership '${userId}': ignoring`, + ); + return []; + } + const membershipOnlyVm = + tilesById.get(userId)?.data ?? + new MembershipOnlyViewModel(member); + tilesById.delete(userId); + + const membershipOnlyTile: TileDescriptor = { + id: userId, + focused: false, + isPresenter: false, + isSpeaker: false, + hasVideo: false, + local: false, + largeBaseSize: false, + data: membershipOnlyVm, + }; + return [membershipOnlyTile]; + } - const userMediaVm = - tilesById.get(userMediaId)?.data ?? - new UserMediaViewModel(userMediaId, member, p, this.encrypted); - tilesById.delete(userMediaId); - - const userMediaTile: TileDescriptor = { - id: userMediaId, - focused: false, - isPresenter: p.isScreenShareEnabled, - isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, - hasVideo: p.isCameraEnabled, - local: p.isLocal, - largeBaseSize: false, - data: userMediaVm, - }; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - const screenShareVm = - tilesById.get(screenShareId)?.data ?? - new ScreenShareViewModel( - screenShareId, - member, - p, - this.encrypted, + const userMediaId = p.identity; + const member = findMatrixMember(this.matrixRoom, userMediaId); + allGhosts &&= member === undefined; + const spokeRecently = + p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; + + // We always start with a local participant with the empty string as + // their ID before we're connected, this is fine and we'll be in + // "all ghosts" mode. + if (userMediaId !== "" && member === undefined) { + logger.warn( + `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, ); - tilesById.delete(screenShareId); - - const screenShareTile: TileDescriptor = { - id: screenShareId, - focused: true, - isPresenter: false, - isSpeaker: false, - hasVideo: true, + } + + const userMediaVm = + tilesById.get(userMediaId)?.data ?? + new UserMediaViewModel(userMediaId, member, p, this.encrypted); + tilesById.delete(userMediaId); + + const userMediaTile: TileDescriptor = { + id: userMediaId, + focused: false, + isPresenter: p.isScreenShareEnabled, + isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, + hasVideo: p.isCameraEnabled, local: p.isLocal, - largeBaseSize: true, - placeNear: userMediaId, - data: screenShareVm, + largeBaseSize: false, + data: userMediaVm, }; - return [userMediaTile, screenShareTile]; - } else { - return [userMediaTile]; - } - }); - // Any tiles left in the map are unused and should be destroyed - for (const t of tilesById.values()) t.data.destroy(); + if (p.isScreenShareEnabled) { + const screenShareId = `${userMediaId}:screen-share`; + const screenShareVm = + tilesById.get(screenShareId)?.data ?? + new ScreenShareViewModel( + screenShareId, + member, + p, + this.encrypted, + ); + tilesById.delete(screenShareId); + + const screenShareTile: TileDescriptor = { + id: screenShareId, + focused: true, + isPresenter: false, + isSpeaker: false, + hasVideo: true, + local: p.isLocal, + largeBaseSize: true, + placeNear: userMediaId, + data: screenShareVm, + }; + return [userMediaTile, screenShareTile]; + } else { + return [userMediaTile]; + } + }); - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? [] : newTiles; - }, [] as TileDescriptor[]), + // Any tiles left in the map are unused and should be destroyed + for (const t of tilesById.values()) t.data.destroy(); + + // If every item is a ghost, that probably means we're still connecting + // and shouldn't bother showing anything yet + return allGhosts ? [] : newTiles; + }, + [] as TileDescriptor[], + ), finalizeValue((ts) => { for (const t of ts) t.data.destroy(); }), ), ); + private get matrixRoom(): MatrixRoom { + return this.rtcSession.room; + } + public constructor( // A call is permanently tied to a single Matrix room and LiveKit room - private readonly matrixRoom: MatrixRoom, + private readonly rtcSession: MatrixRTCSession, private readonly livekitRoom: LivekitRoom, private readonly encrypted: boolean, private readonly connectionState: Observable, @@ -588,25 +675,25 @@ export class CallViewModel extends ViewModel { } export function useCallViewModel( - matrixRoom: MatrixRoom, + rtcSession: MatrixRTCSession, livekitRoom: LivekitRoom, encrypted: boolean, connectionState: ECConnectionState, ): CallViewModel { - const prevMatrixRoom = usePrevious(matrixRoom); + const prevRTCSession = usePrevious(rtcSession); const prevLivekitRoom = usePrevious(livekitRoom); const prevEncrypted = usePrevious(encrypted); const connectionStateObservable = useObservable(connectionState); const vm = useRef(); if ( - matrixRoom !== prevMatrixRoom || + rtcSession !== prevRTCSession || livekitRoom !== prevLivekitRoom || encrypted !== prevEncrypted ) { vm.current?.destroy(); vm.current = new CallViewModel( - matrixRoom, + rtcSession, livekitRoom, encrypted, connectionStateObservable, diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index db11017e1..7417f3f7e 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -110,7 +110,10 @@ abstract class BaseMediaViewModel extends ViewModel { /** * Some participant's media. */ -export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +export type MediaViewModel = + | UserMediaViewModel + | ScreenShareViewModel + | MembershipOnlyViewModel; /** * Some participant's user media. @@ -239,3 +242,15 @@ export class ScreenShareViewModel extends BaseMediaViewModel { ); } } + +/** + * Placeholder for a call membership that does not have a LiveKit participant associated with it. + */ +export class MembershipOnlyViewModel extends ViewModel { + public id: string; + public local = false; + public constructor(public readonly member: RoomMember) { + super(); + this.id = member.userId; + } +} diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index d4a7442ec..13be0b960 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -53,6 +53,7 @@ import { Menu, } from "@vector-im/compound-web"; import { useStateObservable } from "@react-rxjs/core"; +import useRelativeTime from "@nkzw/use-relative-time"; import { Avatar } from "../Avatar"; import styles from "./VideoTile.module.css"; @@ -61,6 +62,7 @@ import { ScreenShareViewModel, MediaViewModel, UserMediaViewModel, + MembershipOnlyViewModel, } from "../state/MediaViewModel"; import { subscribe } from "../state/subscribe"; import { useMergedRefs } from "../useMergedRefs"; @@ -72,7 +74,7 @@ interface TileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - video: TrackReferenceOrPlaceholder; + video?: TrackReferenceOrPlaceholder; member: RoomMember | undefined; videoEnabled: boolean; maximised: boolean; @@ -110,6 +112,10 @@ const Tile = forwardRef( const { t } = useTranslation(); const mergedRef = useMergedRefs(tileRef, ref); + const joinedCallTime = member?.events.member?.getTs() ?? 0; + + const joinedCallAgo = useRelativeTime(joinedCallTime ?? 0); + return ( ( src={member?.getMxcAvatarUrl()} className={styles.avatar} /> - {video.publication !== undefined && ( + {video?.publication !== undefined && ( ( disablePictureInPicture /> )} + {!video && member && joinedCallTime > 0 && ( +
+ {nameTag} joined the call{" "} + + {joinedCallAgo} + {" "} + but is currently unreachable. Are they having connection problems? +
+ )}
{nameTagLeadingIcon} - {nameTag} + {nameTag}{" "} {unencryptedWarning && ( ( ScreenShareTile.displayName = "ScreenShareTile"; +interface MembershipOnlyTileProps { + vm: MembershipOnlyViewModel; + className?: string; + style?: ComponentProps["style"]; + targetWidth: number; + targetHeight: number; + nameTag: string; + displayName: string; + maximised: boolean; + fullscreen: boolean; + onToggleFullscreen: (itemId: string) => void; +} + +const MembershipOnlyTile = subscribe( + ( + { + vm, + className, + style, + targetWidth, + targetHeight, + nameTag, + displayName, + maximised, + fullscreen, + onToggleFullscreen, + }, + ref, + ) => { + const { t } = useTranslation(); + const onClickFullScreen = useCallback( + () => onToggleFullscreen(vm.id), + [onToggleFullscreen, vm], + ); + + const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; + + return ( + + + + } + /> + ); + }, +); + +MembershipOnlyTile.displayName = "MembershipOnlyTile"; + interface Props { vm: MediaViewModel; maximised: boolean; @@ -471,7 +555,7 @@ export const VideoTile = forwardRef( showSpeakingIndicator={showSpeakingIndicator} /> ); - } else { + } else if (vm instanceof ScreenShareViewModel) { return ( ( onToggleFullscreen={onToggleFullscreen} /> ); + } else if (vm instanceof MembershipOnlyViewModel) { + return ( + + ); } }, ); diff --git a/yarn.lock b/yarn.lock index 9121b235e..7827f0a1f 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1912,6 +1912,11 @@ version "3.2.14" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz#acd96c00a881d0f462e1f97a56c73742c8dbc984" +"@nkzw/use-relative-time@^1.1.0": + version "1.1.0" + resolved "https://registry.yarnpkg.com/@nkzw/use-relative-time/-/use-relative-time-1.1.0.tgz#7eb52dc4c7ceaf49a1f05ecfbf725313cf5c1724" + integrity sha512-ogCL62FvScpRpsZUuaN6Jt0xPGRv62atQUNGyMcX+nZs4H5Fs5K1iA3MbSmkJ1y0n/N0RIRLc3VAp8o46lq2CA== + "@nodelib/fs.scandir@2.1.5": version "2.1.5" resolved "https://registry.yarnpkg.com/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz#7619c2eb21b25483f6d167548b4cfd5a7488c3d5" From 2c5b1de6b4e91684fe518f22dbe66ba235ce244d Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Wed, 17 Jul 2024 11:56:45 +0100 Subject: [PATCH 2/4] Use prototype of matrix-js-sdk to send heartbeats and show offline status in UI --- package.json | 4 ++-- src/video-grid/VideoTile.tsx | 42 +++++++++++++++++++++++++++++++++--- yarn.lock | 26 +++++++++++----------- 3 files changed, 54 insertions(+), 18 deletions(-) diff --git a/package.json b/package.json index 1780075fb..a8af62b09 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,7 @@ "i18next-http-backend": "^2.0.0", "livekit-client": "^2.0.2", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#238eea0ef5c82d0a11b8d5cc5c04104d6c94c4c1", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c795c7eb656f2f6136ac4c6b17015c0f000df9a6", "matrix-widget-api": "^1.3.1", "normalize.css": "^8.0.1", "pako": "^2.0.4", @@ -129,4 +129,4 @@ "vite-plugin-svgr": "^4.0.0", "vitest": "^2.0.0" } -} +} \ No newline at end of file diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 13be0b960..9c431cb90 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -112,8 +112,27 @@ const Tile = forwardRef( const { t } = useTranslation(); const mergedRef = useMergedRefs(tileRef, ref); + const [lastActiveTime, setLastActiveTime] = useState( + member?.getLastTypingTime() ?? 0, + ); + const [online, setOnline] = useState(member?.typing ?? false); + + useEffect(() => { + const onTyping = (): void => { + setOnline(member?.typing ?? false); + setLastActiveTime(member?.getLastTypingTime() ?? 0); + }; + + member?.on(RoomMemberEvent.Typing, onTyping); + + return (): void => { + member?.off(RoomMemberEvent.Typing, onTyping); + }; + }, [member]); + const joinedCallTime = member?.events.member?.getTs() ?? 0; + const lastActiveAgo = useRelativeTime(lastActiveTime ?? 0); const joinedCallAgo = useRelativeTime(joinedCallTime ?? 0); return ( @@ -145,11 +164,24 @@ const Tile = forwardRef( )} {!video && member && joinedCallTime > 0 && (
- {nameTag} joined the call{" "} + + {nameTag} ({member.userId}) + {" "} + joined the call{" "} {joinedCallAgo} {" "} - but is currently unreachable. Are they having connection problems? + but is currently unreachable. + {lastActiveTime > 0 && ( + + They were last reachable{" "} + + {lastActiveAgo} + + . + + )}{" "} + Are they having connection problems?
)}
@@ -157,7 +189,11 @@ const Tile = forwardRef(
{nameTagLeadingIcon} - {nameTag}{" "} + 0 ? ` last reachable ${online ? "now" : lastActiveAgo}` : ""}`} + > + {nameTag} + {" "} {unencryptedWarning && ( Date: Fri, 19 Jul 2024 09:07:01 +0100 Subject: [PATCH 3/4] Use Node 20 --- .github/workflows/element-call.yaml | 1 + .github/workflows/lint.yaml | 1 + .github/workflows/test.yaml | 1 + .github/workflows/translations-download.yaml | 1 + .nvmrc | 1 + 5 files changed, 5 insertions(+) create mode 100644 .nvmrc diff --git a/.github/workflows/element-call.yaml b/.github/workflows/element-call.yaml index ad44570f3..5311abe41 100644 --- a/.github/workflows/element-call.yaml +++ b/.github/workflows/element-call.yaml @@ -26,6 +26,7 @@ jobs: uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 with: cache: "yarn" + node-version-file: ".nvmrc" - name: Install dependencies run: "yarn install" - name: Build diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index fdd0857b8..6ce6b00fb 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -12,6 +12,7 @@ jobs: uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 with: cache: "yarn" + node-version-file: ".nvmrc" - name: Install dependencies run: "yarn install" - name: Prettier diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 782050666..0841ed5f0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -14,6 +14,7 @@ jobs: uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 with: cache: "yarn" + node-version-file: ".nvmrc" - name: Install dependencies run: "yarn install" - name: Vitest diff --git a/.github/workflows/translations-download.yaml b/.github/workflows/translations-download.yaml index eb8839c98..9dd4e4ad5 100644 --- a/.github/workflows/translations-download.yaml +++ b/.github/workflows/translations-download.yaml @@ -18,6 +18,7 @@ jobs: - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4 with: cache: "yarn" + node-version-file: ".nvmrc" - name: Install Deps run: "yarn install --frozen-lockfile" diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 000000000..209e3ef4b --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +20 From 7b31c7ec1eeb3941fa383d76685449eeaece3301 Mon Sep 17 00:00:00 2001 From: Hugh Nimmo-Smith Date: Fri, 19 Jul 2024 09:55:38 +0100 Subject: [PATCH 4/4] Lint --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a8af62b09..8d7e254db 100644 --- a/package.json +++ b/package.json @@ -129,4 +129,4 @@ "vite-plugin-svgr": "^4.0.0", "vitest": "^2.0.0" } -} \ No newline at end of file +}