From 91f409e8f4c02c54b3d301f6b801672600669403 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 21 Jul 2021 23:29:08 -0700 Subject: [PATCH 001/291] Add invitee field --- src/client.ts | 5 +++-- src/webrtc/call.ts | 8 ++++++++ src/webrtc/callEventHandler.ts | 4 ++++ src/webrtc/callEventTypes.ts | 1 + 4 files changed, 16 insertions(+), 2 deletions(-) diff --git a/src/client.ts b/src/client.ts index e5424224c3d..7c0766b2c74 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1029,10 +1029,11 @@ export class MatrixClient extends EventEmitter { * The place*Call methods on the returned call can be used to actually place a call * * @param {string} roomId The room the call is to be placed in. + * @param {string} invitee The user to call in the given room. * @return {MatrixCall} the call or null if the browser doesn't support calling. */ - public createCall(roomId: string): MatrixCall { - return createNewMatrixCall(this, roomId); + public createCall(roomId: string, invitee?: string): MatrixCall { + return createNewMatrixCall(this, roomId, { invitee }); } /** diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ed1af7b9005..9466207d1cb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -57,6 +57,7 @@ import { CallFeed } from './callFeed'; interface CallOpts { roomId?: string; + invitee?: string; client?: any; // Fix when client is TSified forceTURN?: boolean; turnServers?: Array; @@ -259,6 +260,7 @@ function genCallID(): string { */ export class MatrixCall extends EventEmitter { roomId: string; + invitee?: string; type: CallType; callId: string; state: CallState; @@ -316,6 +318,7 @@ export class MatrixCall extends EventEmitter { constructor(opts: CallOpts) { super(); this.roomId = opts.roomId; + this.invitee = opts.invitee; this.client = opts.client; this.type = null; this.forceTURN = opts.forceTURN; @@ -1164,6 +1167,10 @@ export class MatrixCall extends EventEmitter { lifetime: CALL_TIMEOUT_MS, } as MCallOfferNegotiate; + if (eventType === EventType.CallInvite && this.invitee) { + content.invitee = this.invitee; + } + // clunky because TypeScript can't folow the types through if we use an expression as the key if (this.state === CallState.CreateOffer) { content.offer = this.peerConn.localDescription; @@ -1801,6 +1808,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO const opts = { client: client, roomId: roomId, + invitee: options.invitee, turnServers: client.getTurnServers(), // call level options forceTURN: client.forceTURN || optionsForceTURN, diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 9d62375e99d..d31e912715f 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -152,6 +152,10 @@ export class CallEventHandler { ); } + if (content.invitee && content.invitee !== this.client.getUserId()) { + return; // This invite was meant for another user in the room + } + const timeUntilTurnCresExpire = this.client.getTurnServersExpiry() - Date.now(); logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms"); call = createNewMatrixCall(this.client, event.getRoomId(), { diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index dce14648533..e79e9d51333 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -25,6 +25,7 @@ export interface MCallOfferNegotiate { description: CallOfferAnswer; lifetime: number; capabilities: CallCapabilities; + invitee?: string; } export interface MCallReplacesTarget { From 76f11bee9e9e6dcd758834e1207f523c4b6a2cc5 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 26 Jul 2021 11:38:18 -0700 Subject: [PATCH 002/291] Fix invitee glare detection and incoming call event --- src/webrtc/call.ts | 2 +- src/webrtc/callEventHandler.ts | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9466207d1cb..7eafef6ec75 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1808,7 +1808,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO const opts = { client: client, roomId: roomId, - invitee: options.invitee, + invitee: options && options.invitee, turnServers: client.getTurnServers(), // call level options forceTURN: client.forceTURN || optionsForceTURN, diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index d31e912715f..b8822084bfe 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -173,7 +173,7 @@ export class CallEventHandler { } call.callId = content.call_id; - call.initWithInvite(event); + const invitePromise = call.initWithInvite(event); this.calls.set(call.callId, call); // if we stashed candidate events for that call ID, play them back now @@ -193,6 +193,7 @@ export class CallEventHandler { if ( call.roomId === thisCall.roomId && thisCall.direction === CallDirection.Outbound && + call.invitee === thisCall.invitee && isCalling ) { existingCall = thisCall; @@ -222,7 +223,9 @@ export class CallEventHandler { call.hangup(CallErrorCode.Replaced, true); } } else { - this.client.emit("Call.incoming", call); + invitePromise.then(() => { + this.client.emit("Call.incoming", call); + }); } } else if (event.getType() === EventType.CallAnswer) { if (!call) { From 1c5101aa1a7f92871cd25d58b2c79e6059ab87ba Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 4 Aug 2021 18:23:21 -0700 Subject: [PATCH 003/291] Add ice disconnected timeout --- src/webrtc/call.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7eafef6ec75..0640110ac10 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -291,6 +291,7 @@ export class MatrixCall extends EventEmitter { private opponentPartyId: string; private opponentCaps: CallCapabilities; private inviteTimeout: NodeJS.Timeout; // in the browser it's 'number' + private iceDisconnectedTimeout: NodeJS.Timeout; // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold // This flag represents whether we want the other party to be on hold @@ -1269,9 +1270,14 @@ export class MatrixCall extends EventEmitter { // ideally we'd consider the call to be connected when we get media but // chrome doesn't implement any of the 'onstarted' events yet if (this.peerConn.iceConnectionState == 'connected') { + clearTimeout(this.iceDisconnectedTimeout); this.setState(CallState.Connected); } else if (this.peerConn.iceConnectionState == 'failed') { this.hangup(CallErrorCode.IceFailed, false); + } else if (this.peerConn.iceConnectionState == 'disconnected') { + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); } }; From 2cd5c813acb92a5c531d056e305533c71f3c8d05 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 5 Aug 2021 18:22:29 -0700 Subject: [PATCH 004/291] Add local media stream functions to client --- src/client.ts | 38 ++++++++++++++++++++++++++++++++++- src/webrtc/call.ts | 49 ++++++++++++++++++++++++---------------------- 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/src/client.ts b/src/client.ts index 7c0766b2c74..90aadf42524 100644 --- a/src/client.ts +++ b/src/client.ts @@ -23,7 +23,7 @@ import { EventEmitter } from "events"; import { SyncApi } from "./sync"; import { EventStatus, IDecryptOptions, MatrixEvent } from "./models/event"; import { StubStore } from "./store/stub"; -import { createNewMatrixCall, MatrixCall } from "./webrtc/call"; +import { createNewMatrixCall, MatrixCall, ConstraintsType, getUserMediaContraints } from "./webrtc/call"; import { Filter } from "./filter"; import { CallEventHandler } from './webrtc/callEventHandler'; import * as utils from './utils'; @@ -458,6 +458,8 @@ export class MatrixClient extends EventEmitter { public iceCandidatePoolSize = 0; // XXX: Intended private, used in code. public idBaseUrl: string; public baseUrl: string; + private localAVStreamType: ConstraintsType; + private localAVStream: MediaStream; // Note: these are all `protected` to let downstream consumers make mistakes if they want to. // We don't technically support this usage, but have reasons to do this. @@ -1024,6 +1026,40 @@ export class MatrixClient extends EventEmitter { this.supportsCallTransfer = support; } + public async getLocalVideoStream() { + if (this.localAVStreamType === ConstraintsType.Video) { + return this.localAVStream.clone(); + } + + const constraints = getUserMediaContraints(ConstraintsType.Video); + logger.log("Getting user media with constraints", constraints); + const mediaStream = await navigator.mediaDevices.getUserMedia(constraints); + this.localAVStreamType = ConstraintsType.Video; + this.localAVStream = mediaStream; + return mediaStream; + } + + public async getLocalAudioStream() { + if (this.localAVStreamType === ConstraintsType.Audio) { + return this.localAVStream.clone(); + } + + const constraints = getUserMediaContraints(ConstraintsType.Audio); + logger.log("Getting user media with constraints", constraints); + const mediaStream = await navigator.mediaDevices.getUserMedia(constraints); + this.localAVStreamType = ConstraintsType.Audio; + this.localAVStream = mediaStream; + return mediaStream; + } + + public stopLocalMediaStream() { + if (this.localAVStream) { + for (const track of this.localAVStream.getTracks()) { + track.stop(); + } + } + } + /** * Creates a new call. * The place*Call methods on the returned call can be used to actually place a call diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0640110ac10..583b45e9de6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -201,7 +201,7 @@ export enum CallErrorCode { Transfered = 'transferred', } -enum ConstraintsType { +export enum ConstraintsType { Audio = "audio", Video = "video", } @@ -281,8 +281,6 @@ export class MatrixCall extends EventEmitter { private localAVStream: MediaStream; private inviteOrAnswerSent: boolean; private waitForLocalAVStream: boolean; - // XXX: I don't know why this is called 'config'. - private config: MediaStreamConstraints; private successor: MatrixCall; private opponentMember: RoomMember; private opponentVersion: number; @@ -362,9 +360,8 @@ export class MatrixCall extends EventEmitter { async placeVoiceCall() { logger.debug("placeVoiceCall"); this.checkForErrorListener(); - const constraints = getUserMediaContraints(ConstraintsType.Audio); this.type = CallType.Voice; - await this.placeCallWithConstraints(constraints); + await this.placeCall(ConstraintsType.Audio); } /** @@ -374,9 +371,8 @@ export class MatrixCall extends EventEmitter { async placeVideoCall() { logger.debug("placeVideoCall"); this.checkForErrorListener(); - const constraints = getUserMediaContraints(ConstraintsType.Video); this.type = CallType.Video; - await this.placeCallWithConstraints(constraints); + await this.placeCall(ConstraintsType.Video); } /** @@ -406,8 +402,7 @@ export class MatrixCall extends EventEmitter { } logger.debug("Got screen stream, requesting audio stream..."); - const audioConstraints = getUserMediaContraints(ConstraintsType.Audio); - this.placeCallWithConstraints(audioConstraints); + this.placeCall(ConstraintsType.Audio); } catch (err) { this.emit(CallEvent.Error, new CallError( @@ -587,17 +582,18 @@ export class MatrixCall extends EventEmitter { logger.debug(`Answering call ${this.callId} of type ${this.type}`); if (!this.localAVStream && !this.waitForLocalAVStream) { - const constraints = getUserMediaContraints( - this.type == CallType.Video ? - ConstraintsType.Video: - ConstraintsType.Audio, - ); - logger.log("Getting user media with constraints", constraints); this.setState(CallState.WaitLocalMedia); this.waitForLocalAVStream = true; try { - const mediaStream = await navigator.mediaDevices.getUserMedia(constraints); + let mediaStream: MediaStream; + + if (this.type === CallType.Voice) { + mediaStream = await this.client.getLocalAudioStream(); + } else { + mediaStream = await this.client.getLocalVideoStream(); + } + this.waitForLocalAVStream = false; this.gotUserMediaForAnswer(mediaStream); } catch (e) { @@ -1519,8 +1515,10 @@ export class MatrixCall extends EventEmitter { logger.debug(`stopAllMedia (stream=${this.localAVStream})`); for (const feed of this.feeds) { - for (const track of feed.stream.getTracks()) { - track.stop(); + if (!feed.isLocal()) { + for (const track of feed.stream.getTracks()) { + track.stop(); + } } } } @@ -1579,13 +1577,11 @@ export class MatrixCall extends EventEmitter { } } - private async placeCallWithConstraints(constraints: MediaStreamConstraints) { - logger.log("Getting user media with constraints", constraints); + private async placeCall(constraintsType: ConstraintsType) { // XXX Find a better way to do this this.client.callEventHandler.calls.set(this.callId, this); this.setState(CallState.WaitLocalMedia); this.direction = CallDirection.Outbound; - this.config = constraints; // make sure we have valid turn creds. Unless something's gone wrong, it should // poll and keep the credentials valid so this should be instant. @@ -1599,7 +1595,14 @@ export class MatrixCall extends EventEmitter { this.peerConn = this.createPeerConnection(); try { - const mediaStream = await navigator.mediaDevices.getUserMedia(constraints); + let mediaStream: MediaStream; + + if (constraintsType === ConstraintsType.Audio) { + mediaStream = await this.client.getLocalAudioStream(); + } else { + mediaStream = await this.client.getLocalVideoStream(); + } + this.gotUserMediaForInvite(mediaStream); } catch (e) { this.getUserMediaFailed(e); @@ -1693,7 +1696,7 @@ function setTracksEnabled(tracks: Array, enabled: boolean) { } } -function getUserMediaContraints(type: ConstraintsType) { +export function getUserMediaContraints(type: ConstraintsType) { const isWebkit = !!navigator.webkitGetUserMedia; switch (type) { From 154e5c45a6592451098e075a46ff07ca6497842a Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 9 Aug 2021 11:02:09 -0700 Subject: [PATCH 005/291] Clear localAVStream when stopping local media stream. --- src/client.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/client.ts b/src/client.ts index 90aadf42524..f8293857228 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1057,6 +1057,9 @@ export class MatrixClient extends EventEmitter { for (const track of this.localAVStream.getTracks()) { track.stop(); } + + this.localAVStreamType = null; + this.localAVStream = null; } } From 782fbb115f7998530bfe816f41d2887101c0e2bd Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 20 Aug 2021 14:42:41 -0700 Subject: [PATCH 006/291] Stop all media on hangup --- src/webrtc/call.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 09412e089d5..04cf53c2593 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1774,10 +1774,8 @@ export class MatrixCall extends EventEmitter { logger.debug(`stopAllMedia (stream=${this.localUsermediaStream})`); for (const feed of this.feeds) { - if (!feed.isLocal()) { - for (const track of feed.stream.getTracks()) { - track.stop(); - } + for (const track of feed.stream.getTracks()) { + track.stop(); } } } From 19302ea4fbafd00b7be96f310bc5e35255799c4f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 31 Aug 2021 16:10:37 -0700 Subject: [PATCH 007/291] Fix initWithInvitePromise --- src/webrtc/callEventHandler.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 52ace039b84..7af3242070a 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -226,9 +226,7 @@ export class CallEventHandler { call.hangup(CallErrorCode.Replaced, true); } } else { - initWithInvitePromise.then(() => { - this.client.emit("Call.incoming", call); - }); + this.client.emit("Call.incoming", call); } return; } else if (type === EventType.CallCandidates) { From 03dfab1282f7ce65aab8025275b4360456dc828d Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 2 Sep 2021 13:01:43 -0700 Subject: [PATCH 008/291] Export CallFeed --- src/matrix.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/matrix.ts b/src/matrix.ts index 97709e314ab..8669a686b55 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -53,6 +53,11 @@ export { setVideoInput as setMatrixCallVideoInput, } from "./webrtc/call"; +// TODO: This export is temporary and is only used for the local call feed for conference calls +// Ideally conference calls will become a first-class concept and we will have a local call feed with +// a lifecycle that matches the conference call, not individual calls to members. +export { CallFeed } from "./webrtc/callFeed"; + // expose the underlying request object so different environments can use // different request libs (e.g. request or browser-request) let requestInstance; From 60e168806d05b426cd7a4469c711a85464fcb6d9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 2 Sep 2021 13:27:13 -0700 Subject: [PATCH 009/291] Properly dispose of call feeds --- src/webrtc/call.ts | 5 +++++ src/webrtc/callFeed.ts | 7 ++++++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 5c151eecafc..4ac19d23f65 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -555,6 +555,10 @@ export class MatrixCall extends EventEmitter { } private deleteAllFeeds(): void { + for (const feed of this.feeds) { + feed.dispose(); + } + this.feeds = []; this.emit(CallEvent.FeedsChanged, this.feeds); } @@ -568,6 +572,7 @@ export class MatrixCall extends EventEmitter { return; } + feed.dispose(); this.feeds.splice(this.feeds.indexOf(feed), 1); this.emit(CallEvent.FeedsChanged, this.feeds); } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 6ebadb27314..d4287b1deb8 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -36,6 +36,7 @@ export class CallFeed extends EventEmitter { private frequencyBinCount: Float32Array; private speakingThreshold = SPEAKING_THRESHOLD; private speaking = false; + private volumeLooperTimeout: number; constructor( public stream: MediaStream, @@ -166,7 +167,7 @@ export class CallFeed extends EventEmitter { private volumeLooper(): void { if (!this.analyser) return; - setTimeout(() => { + this.volumeLooperTimeout = setTimeout(() => { if (!this.measuringVolumeActivity) return; this.analyser.getFloatFrequencyData(this.frequencyBinCount); @@ -188,4 +189,8 @@ export class CallFeed extends EventEmitter { this.volumeLooper(); }, POLLING_INTERVAL); } + + public dispose(): void { + clearTimeout(this.volumeLooperTimeout); + } } From 4fe115b2c425720ee5464b3ea880016bc0f49f8b Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 8 Sep 2021 13:27:38 -0700 Subject: [PATCH 010/291] Add initial group call logic --- src/client.ts | 28 +- src/matrix.ts | 1 + src/webrtc/call.ts | 14 + src/webrtc/callFeed.ts | 2 +- src/webrtc/groupCall.ts | 763 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 802 insertions(+), 6 deletions(-) create mode 100644 src/webrtc/groupCall.ts diff --git a/src/client.ts b/src/client.ts index a88881612b3..d4d79aeb275 100644 --- a/src/client.ts +++ b/src/client.ts @@ -23,7 +23,7 @@ import { EventEmitter } from "events"; import { ISyncStateData, SyncApi } from "./sync"; import { EventStatus, IContent, IDecryptOptions, IEvent, MatrixEvent } from "./models/event"; import { StubStore } from "./store/stub"; -import { createNewMatrixCall, MatrixCall, ConstraintsType, getUserMediaContraints } from "./webrtc/call"; +import { createNewMatrixCall, MatrixCall, ConstraintsType, getUserMediaContraints, CallType } from "./webrtc/call"; import { Filter, IFilterDefinition } from "./filter"; import { CallEventHandler } from './webrtc/callEventHandler'; import * as utils from './utils'; @@ -144,6 +144,7 @@ import { IHierarchyRoom, ISpaceSummaryEvent, ISpaceSummaryRoom } from "./@types/ import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; +import { GroupCall, GroupCallEvent } from "./webrtc/groupCall"; export type Store = IStore; export type SessionStore = WebStorageSessionStore; @@ -1315,6 +1316,23 @@ export class MatrixClient extends EventEmitter { return createNewMatrixCall(this, roomId, { invitee }); } + /** + * Creates a new group call. + * + * @param {string} roomId The room the call is to be placed in. + * @return {GroupCall} the call or null if the browser doesn't support calling. + */ + public createGroupCall( + roomId: string, + type: CallType, + dataChannelsEnabled?: boolean, + dataChannelOptions?: RTCDataChannelInit, + ): GroupCall { + const groupCall = new GroupCall(this, roomId, type, dataChannelsEnabled, dataChannelOptions); + this.reEmitter.reEmit(groupCall, Object.values(GroupCallEvent)); + return groupCall; + } + /** * Get the current sync state. * @return {?SyncState} the sync state, which may be null. @@ -6153,11 +6171,11 @@ export class MatrixClient extends EventEmitter { public register( username: string, password: string, - sessionId: string, + sessionId: string | null, auth: any, - bindThreepids: any, - guestAccessToken: string, - inhibitLogin: boolean, + bindThreepids?: any, + guestAccessToken?: string, + inhibitLogin?: boolean, callback?: Callback, ): Promise { // TODO: Types (many) // backwards compat diff --git a/src/matrix.ts b/src/matrix.ts index 8669a686b55..d915c231050 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -51,6 +51,7 @@ export { createNewMatrixCall, setAudioInput as setMatrixCallAudioInput, setVideoInput as setMatrixCallVideoInput, + CallType, } from "./webrtc/call"; // TODO: This export is temporary and is only used for the local call feed for conference calls diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4ac19d23f65..bd689004b0d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -128,6 +128,8 @@ export enum CallEvent { FeedsChanged = 'feeds_changed', AssertedIdentityChanged = 'asserted_identity_changed', + + DataChannel = 'datachannel', } export enum CallErrorCode { @@ -368,6 +370,13 @@ export class MatrixCall extends EventEmitter { await this.placeCallWithConstraints(ConstraintsType.Video); } + public createDataChannel(label: string, options: RTCDataChannelInit) { + logger.debug("createDataChannel"); + const dataChannel = this.peerConn.createDataChannel(label, options); + this.emit(CallEvent.DataChannel, dataChannel); + return dataChannel; + } + public getOpponentMember(): RoomMember { return this.opponentMember; } @@ -1525,6 +1534,10 @@ export class MatrixCall extends EventEmitter { stream.addEventListener("removetrack", () => this.deleteFeedByStream(stream)); }; + private onDataChannel = (ev: RTCDataChannelEvent): void => { + this.emit(CallEvent.DataChannel, ev.channel); + }; + /** * This method removes all video/rtx codecs from screensharing video * transceivers. This is necessary since they can cause problems. Without @@ -1883,6 +1896,7 @@ export class MatrixCall extends EventEmitter { pc.addEventListener('icegatheringstatechange', this.onIceGatheringStateChange); pc.addEventListener('track', this.onTrack); pc.addEventListener('negotiationneeded', this.onNegotiationNeeded); + pc.addEventListener('datachannel', this.onDataChannel); return pc; } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index d4287b1deb8..29a8a76687e 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -35,7 +35,7 @@ export class CallFeed extends EventEmitter { private analyser: AnalyserNode; private frequencyBinCount: Float32Array; private speakingThreshold = SPEAKING_THRESHOLD; - private speaking = false; + public speaking = false; private volumeLooperTimeout: number; constructor( diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts new file mode 100644 index 00000000000..0d3efc74c1c --- /dev/null +++ b/src/webrtc/groupCall.ts @@ -0,0 +1,763 @@ +import EventEmitter from "events"; +import { CallFeed, CallFeedEvent } from "./callFeed"; +import { MatrixClient } from "../client"; +import { randomString } from "../randomstring"; +import { CallErrorCode, CallEvent, CallType, MatrixCall } from "./call"; +import { RoomMember } from "../models/room-member"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { Room } from "../models/room"; +import { logger } from "../logger"; + +export enum GroupCallEvent { + ActiveSpeakerChanged = "active_speaker_changed", +} + +const CONF_ROOM = "me.robertlong.conf"; +const CONF_PARTICIPANT = "me.robertlong.conf.participant"; +const PARTICIPANT_TIMEOUT = 1000 * 15; +const SPEAKING_THRESHOLD = -80; +const ACTIVE_SPEAKER_INTERVAL = 1000; +const ACTIVE_SPEAKER_SAMPLES = 8; + +export class GroupCallParticipant extends EventEmitter { + public feeds: CallFeed[] = []; + public activeSpeaker: boolean; + public activeSpeakerSamples: number[]; + public dataChannel?: RTCDataChannel; + + constructor( + private groupCall: GroupCall, + public member: RoomMember, + // The session id is used to re-initiate calls if the user's participant + // session id has changed + public sessionId: string, + public call?: MatrixCall, + ) { + super(); + + this.activeSpeakerSamples = Array(ACTIVE_SPEAKER_SAMPLES).fill( + -Infinity, + ); + + if (this.call) { + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + } + } + + public replaceCall(call: MatrixCall, sessionId: string) { + if (this.call) { + this.call.hangup(CallErrorCode.Replaced, false); + this.call.removeListener(CallEvent.State, this.onCallStateChanged); + this.call.removeListener( + CallEvent.FeedsChanged, + this.onCallFeedsChanged, + ); + this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); + this.call.removeListener(CallEvent.Hangup, this.onCallHangup); + this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); + } + + this.call = call; + this.member = call.getOpponentMember(); + this.activeSpeaker = false; + this.sessionId = sessionId; + + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + this.call.on(CallEvent.DataChannel, this.onCallDataChannel); + } + + public get usermediaFeed() { + return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get usermediaStream(): MediaStream { + return this.usermediaFeed?.stream; + } + + public isAudioMuted(): boolean { + const feed = this.usermediaFeed; + + if (!feed) { + return true; + } + + return feed.isAudioMuted(); + } + + public isVideoMuted(): boolean { + const feed = this.usermediaFeed; + + if (!feed) { + return true; + } + + return feed.isVideoMuted(); + } + + private onCallStateChanged = (state) => { + const call = this.call; + const audioMuted = this.groupCall.localParticipant.isAudioMuted(); + + if ( + call.localUsermediaStream && + call.isMicrophoneMuted() !== audioMuted + ) { + call.setMicrophoneMuted(audioMuted); + } + + const videoMuted = this.groupCall.localParticipant.isVideoMuted(); + + if ( + call.localUsermediaStream && + call.isLocalVideoMuted() !== videoMuted + ) { + call.setLocalVideoMuted(videoMuted); + } + + this.groupCall.emit( + "debugstate", + this.member.userId, + this.call.callId, + state, + ); + }; + + onCallFeedsChanged = () => { + const oldFeeds = this.feeds; + const newFeeds = this.call.getRemoteFeeds(); + + this.feeds = []; + + for (const feed of newFeeds) { + if (oldFeeds.includes(feed)) { + continue; + } + + this.addCallFeed(feed); + } + }; + + onCallReplaced = (newCall) => { + // TODO: Should we always reuse the sessionId? + this.replaceCall(newCall, this.sessionId); + this.groupCall.emit("call", newCall); + this.groupCall.emit("participants_changed"); + }; + + onCallHangup = () => { + if (this.call.hangupReason === "replaced") { + return; + } + + const participantIndex = this.groupCall.participants.indexOf(this); + + if (participantIndex === -1) { + return; + } + + this.groupCall.participants.splice(participantIndex, 1); + + if ( + this.groupCall.activeSpeaker === this && + this.groupCall.participants.length > 0 + ) { + this.groupCall.activeSpeaker = this.groupCall.participants[0]; + this.groupCall.activeSpeaker.activeSpeaker = true; + } + + this.groupCall.emit("participants_changed"); + }; + + addCallFeed(callFeed: CallFeed) { + if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { + callFeed.setSpeakingThreshold(SPEAKING_THRESHOLD); + callFeed.measureVolumeActivity(true); + callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + callFeed.on( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + callFeed.on( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); + this.onCallFeedMuteStateChanged( + this.isAudioMuted(), + this.isVideoMuted(), + ); + } + + this.feeds.push(callFeed); + } + + onCallFeedSpeaking = (speaking: boolean) => { + this.emit("speaking"); + }; + + onCallFeedVolumeChanged = (maxVolume: number) => { + this.activeSpeakerSamples.shift(); + this.activeSpeakerSamples.push(maxVolume); + this.emit("volume_changed", maxVolume); + }; + + onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { + if (audioMuted) { + this.activeSpeakerSamples = Array(ACTIVE_SPEAKER_SAMPLES).fill( + -Infinity, + ); + } + + this.emit("mute_state_changed", audioMuted, videoMuted); + }; + + onCallDataChannel = (dataChannel: RTCDataChannel) => { + this.dataChannel = dataChannel; + this.emit("datachannel"); + }; +} + +export class GroupCall extends EventEmitter { + public entered = false; + public activeSpeaker: GroupCallParticipant; + public localParticipant: GroupCallParticipant; + public participants: GroupCallParticipant[] = []; + public room: Room; + + private speakerMap: Map = new Map(); + private presenceLoopTimeout?: number; + private activeSpeakerLoopTimeout: number; + + constructor( + private client: MatrixClient, + roomId: string, + public type: CallType, + private dataChannelsEnabled?: boolean, + private dataChannelOptions?: RTCDataChannelInit, + ) { + super(); + + this.room = this.client.getRoom(roomId); + } + + async initLocalParticipant() { + if (this.localParticipant) { + return this.localParticipant; + } + + let stream; + + if (this.type === CallType.Video) { + stream = await this.client.getLocalVideoStream(); + } else { + stream = await this.client.getLocalAudioStream(); + } + + const userId = this.client.getUserId(); + + const localCallFeed = new CallFeed( + stream, + userId, + SDPStreamMetadataPurpose.Usermedia, + this.client, + this.room.roomId, + false, + false, + ); + + const member = this.room.getMember(userId); + + this.localParticipant = new GroupCallParticipant( + this, + member, + randomString(16), + ); + this.localParticipant.addCallFeed(localCallFeed); + + return this.localParticipant; + } + + async enter() { + if (!this.localParticipant) { + await this.initLocalParticipant(); + } + + // Ensure that this room is marked as a conference room so clients can react appropriately + const activeConf = this.room.currentState + .getStateEvents(CONF_ROOM, "") + ?.getContent()?.active; + + if (!activeConf) { + this.sendStateEventWithRetry( + this.room.roomId, + CONF_ROOM, + { active: true }, + "", + ); + } + + this.activeSpeaker = this.localParticipant; + this.participants.push(this.localParticipant); + + // Announce to the other room members that we have entered the room. + // Continue doing so every PARTICIPANT_TIMEOUT ms + this.onPresenceLoop(); + + this.entered = true; + + this.processInitialCalls(); + + // Set up participants for the members currently in the room. + // Other members will be picked up by the RoomState.members event. + const initialMembers = this.room.getMembers(); + + for (const member of initialMembers) { + this.onMemberChanged(member); + } + + this.client.on("RoomState.members", this.onRoomStateMembers); + this.client.on("Call.incoming", this.onIncomingCall); + + this.emit("entered"); + this.emit("participants_changed"); + this.onActiveSpeakerLoop(); + } + + leave() { + this.localParticipant = null; + this.client.stopLocalMediaStream(); + + if (!this.entered) { + return; + } + + const userId = this.client.getUserId(); + const currentMemberState = this.room.currentState.getStateEvents( + "m.room.member", + userId, + ); + + this.sendStateEventWithRetry( + this.room.roomId, + "m.room.member", + { + ...currentMemberState.getContent(), + [CONF_PARTICIPANT]: null, + }, + userId, + ); + + for (const participant of this.participants) { + if (participant.call) { + participant.call.hangup(CallErrorCode.UserHangup, false); + } + } + + this.entered = false; + this.participants = []; + this.activeSpeaker = null; + this.speakerMap.clear(); + clearTimeout(this.presenceLoopTimeout); + clearTimeout(this.activeSpeakerLoopTimeout); + + this.client.removeListener( + "RoomState.members", + this.onRoomStateMembers, + ); + this.client.removeListener("Call.incoming", this.onIncomingCall); + + this.emit("participants_changed"); + this.emit("left"); + } + + isLocalVideoMuted() { + if (this.localParticipant) { + return this.localParticipant.isVideoMuted(); + } + + return true; + } + + isMicrophoneMuted() { + if (this.localParticipant) { + return this.localParticipant.isAudioMuted(); + } + + return true; + } + + setMicrophoneMuted(muted) { + if (this.localParticipant) { + for (const { stream } of this.localParticipant.feeds) { + for (const track of stream.getTracks()) { + if (track.kind === "audio") { + track.enabled = !muted; + } + } + } + } + + for (const { call } of this.participants) { + if ( + call && + call.localUsermediaStream && + call.isMicrophoneMuted() !== muted + ) { + call.setMicrophoneMuted(muted); + } + } + + this.emit("participants_changed"); + this.emit("audio_mute_state_changed"); + } + + setLocalVideoMuted(muted) { + if (this.localParticipant) { + for (const { stream } of this.localParticipant.feeds) { + for (const track of stream.getTracks()) { + if (track.kind === "video") { + track.enabled = !muted; + } + } + } + } + + for (const { call } of this.participants) { + if ( + call && + call.localUsermediaStream && + call.isLocalVideoMuted() !== muted + ) { + call.setLocalVideoMuted(muted); + } + } + + this.emit("participants_changed"); + this.emit("video_mute_state_changed"); + } + + public get localUsermediaFeed(): CallFeed { + return this.localParticipant?.usermediaFeed; + } + + public get localUsermediaStream(): MediaStream { + return this.localParticipant?.usermediaStream; + } + + /** + * Call presence + */ + + onPresenceLoop = () => { + const userId = this.client.getUserId(); + const currentMemberState = this.room.currentState.getStateEvents( + "m.room.member", + userId, + ); + + this.sendStateEventWithRetry( + this.room.roomId, + "m.room.member", + { + ...currentMemberState.getContent(), + [CONF_PARTICIPANT]: { + sessionId: this.localParticipant.sessionId, + expiresAt: new Date().getTime() + PARTICIPANT_TIMEOUT * 2, + }, + }, + userId, + ); + + const now = new Date().getTime(); + + for (const participant of this.participants) { + if (participant === this.localParticipant) { + continue; + } + + const memberStateEvent = this.room.currentState.getStateEvents( + "m.room.member", + participant.member.userId, + ); + + const memberStateContent = memberStateEvent.getContent(); + + if ( + !memberStateContent || + !memberStateContent[CONF_PARTICIPANT] || + typeof memberStateContent[CONF_PARTICIPANT] !== "object" || + (memberStateContent[CONF_PARTICIPANT].expiresAt && + memberStateContent[CONF_PARTICIPANT].expiresAt < now) + ) { + this.emit( + "debugstate", + participant.member.userId, + null, + "inactive", + ); + + if (participant.call) { + // NOTE: This should remove the participant on the next tick + // since matrix-js-sdk awaits a promise before firing user_hangup + participant.call.hangup(CallErrorCode.UserHangup, false); + } + } + } + + this.presenceLoopTimeout = setTimeout( + this.onPresenceLoop, + PARTICIPANT_TIMEOUT, + ); + }; + + /** + * Call Setup + * + * There are two different paths for calls to be created: + * 1. Incoming calls triggered by the Call.incoming event. + * 2. Outgoing calls to the initial members of a room or new members + * as they are observed by the RoomState.members event. + */ + + processInitialCalls() { + const calls = this.client.callEventHandler.calls.values(); + + for (const call of calls) { + this.onIncomingCall(call); + } + } + + onIncomingCall = (call: MatrixCall) => { + // The incoming calls may be for another room, which we will ignore. + if (call.roomId !== this.room.roomId) { + return; + } + + if (call.state !== "ringing") { + logger.warn("Incoming call no longer in ringing state. Ignoring."); + return; + } + + const opponentMember = call.getOpponentMember(); + + const memberStateEvent = this.room.currentState.getStateEvents( + "m.room.member", + opponentMember.userId, + ); + + const memberStateContent = memberStateEvent.getContent(); + + if (!memberStateContent || !memberStateContent[CONF_PARTICIPANT]) { + call.reject(); + return; + } + + const { sessionId } = memberStateContent[CONF_PARTICIPANT]; + + // Check if the user calling has an existing participant and use this call instead. + const existingParticipant = this.participants.find( + (participant) => participant.member.userId === opponentMember.userId, + ); + + let participant; + + if (existingParticipant) { + participant = existingParticipant; + // This also fires the hangup event and triggers those side-effects + existingParticipant.replaceCall(call, sessionId); + } else { + participant = new GroupCallParticipant( + this, + opponentMember, + sessionId, + call, + ); + this.participants.push(participant); + } + + call.answer(); + + this.emit("call", call); + this.emit("participants_changed"); + }; + + onRoomStateMembers = (_event, _state, member) => { + this.onMemberChanged(member); + }; + + onMemberChanged = (member) => { + // The member events may be received for another room, which we will ignore. + if (member.roomId !== this.room.roomId) { + return; + } + + // Don't process your own member. + const localUserId = this.client.getUserId(); + + if (member.userId === localUserId) { + return; + } + + // Get the latest member participant state event. + const memberStateEvent = this.room.currentState.getStateEvents( + "m.room.member", + member.userId, + ); + const memberStateContent = memberStateEvent.getContent(); + + if (!memberStateContent) { + return; + } + + const participantInfo = memberStateContent[CONF_PARTICIPANT]; + + if (!participantInfo || typeof participantInfo !== "object") { + return; + } + + const { expiresAt, sessionId } = participantInfo; + + // If the participant state has expired, ignore this user. + const now = new Date().getTime(); + + if (expiresAt < now) { + this.emit("debugstate", member.userId, null, "inactive"); + return; + } + + // If there is an existing participant for this member check the session id. + // If the session id changed then we can hang up the old call and start a new one. + // Otherwise, ignore the member change event because we already have an active participant. + let participant = this.participants.find( + (p) => p.member.userId === member.userId, + ); + + if (participant) { + if (participant.sessionId !== sessionId) { + this.emit("debugstate", member.userId, null, "inactive"); + participant.call.hangup(CallErrorCode.Replaced, false); + } else { + return; + } + } + + // Only initiate a call with a user who has a userId that is lexicographically + // less than your own. Otherwise, that user will call you. + if (member.userId < localUserId) { + this.emit("debugstate", member.userId, null, "waiting for invite"); + return; + } + + const call = this.client.createCall(this.room.roomId, member.userId); + + if (participant) { + participant.replaceCall(call, sessionId); + } else { + participant = new GroupCallParticipant( + this, + member, + sessionId, + call, + ); + // TODO: Should we wait until the call has been answered to push the participant? + // Or do we hide the participant until their stream is live? + // Does hiding a participant without a stream present a privacy problem because + // a participant without a stream can still listen in on other user's streams? + this.participants.push(participant); + } + + let callPromise; + + if (this.type === CallType.Video) { + callPromise = call.placeVideoCall(); + } else { + callPromise = call.placeVoiceCall(); + } + + callPromise.then(() => { + if (this.dataChannelsEnabled) { + call.createDataChannel("datachannel", this.dataChannelOptions); + } + + this.emit("call", call); + }); + + this.emit("participants_changed"); + }; + + onActiveSpeakerLoop = () => { + let topAvg; + let nextActiveSpeaker; + + for (const participant of this.participants) { + let total = 0; + + for (let i = 0; i < participant.activeSpeakerSamples.length; i++) { + const volume = participant.activeSpeakerSamples[i]; + total += Math.max(volume, SPEAKING_THRESHOLD); + } + + const avg = total / ACTIVE_SPEAKER_SAMPLES; + + if (!topAvg || avg > topAvg) { + topAvg = avg; + nextActiveSpeaker = participant.member; + } + } + + if (nextActiveSpeaker && topAvg > SPEAKING_THRESHOLD) { + if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) { + this.activeSpeaker.activeSpeaker = false; + nextActiveSpeaker.activeSpeaker = true; + this.activeSpeaker = nextActiveSpeaker; + this.emit("participants_changed"); + } + } + + this.activeSpeakerLoopTimeout = setTimeout( + this.onActiveSpeakerLoop, + ACTIVE_SPEAKER_INTERVAL, + ); + }; + + /** + * Utils + */ + + // TODO: move this elsewhere or get rid of the retry logic. Do we need it? + sendStateEventWithRetry( + roomId, + eventType, + content, + stateKey, + callback = undefined, + maxAttempts = 5, + ) { + const sendStateEventWithRetry = async (attempt = 0) => { + try { + return await this.client.sendStateEvent( + roomId, + eventType, + content, + stateKey, + callback, + ); + } catch (error) { + if (attempt >= maxAttempts) { + throw error; + } + + await new Promise((resolve) => setTimeout(resolve, 5)); + + return sendStateEventWithRetry(attempt + 1); + } + }; + + return sendStateEventWithRetry(); + } +} From 2add1fcbcb92a86cde4581664f0afd8d854420ab Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 8 Sep 2021 14:37:21 -0700 Subject: [PATCH 011/291] Clean up group call events --- src/client.ts | 6 +- src/webrtc/groupCall.ts | 137 ++++++++++++++++++++-------------------- 2 files changed, 70 insertions(+), 73 deletions(-) diff --git a/src/client.ts b/src/client.ts index d4d79aeb275..fdd271c6ef5 100644 --- a/src/client.ts +++ b/src/client.ts @@ -144,7 +144,7 @@ import { IHierarchyRoom, ISpaceSummaryEvent, ISpaceSummaryRoom } from "./@types/ import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; -import { GroupCall, GroupCallEvent } from "./webrtc/groupCall"; +import { GroupCall } from "./webrtc/groupCall"; export type Store = IStore; export type SessionStore = WebStorageSessionStore; @@ -1328,9 +1328,7 @@ export class MatrixClient extends EventEmitter { dataChannelsEnabled?: boolean, dataChannelOptions?: RTCDataChannelInit, ): GroupCall { - const groupCall = new GroupCall(this, roomId, type, dataChannelsEnabled, dataChannelOptions); - this.reEmitter.reEmit(groupCall, Object.values(GroupCallEvent)); - return groupCall; + return new GroupCall(this, roomId, type, dataChannelsEnabled, dataChannelOptions); } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0d3efc74c1c..3ff6f011cac 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -2,14 +2,20 @@ import EventEmitter from "events"; import { CallFeed, CallFeedEvent } from "./callFeed"; import { MatrixClient } from "../client"; import { randomString } from "../randomstring"; -import { CallErrorCode, CallEvent, CallType, MatrixCall } from "./call"; +import { CallErrorCode, CallEvent, CallState, CallType, MatrixCall } from "./call"; import { RoomMember } from "../models/room-member"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { Room } from "../models/room"; import { logger } from "../logger"; +import { Callback } from "../client"; +import { ReEmitter } from "../ReEmitter"; export enum GroupCallEvent { + Entered = "entered", + Left = "left", ActiveSpeakerChanged = "active_speaker_changed", + ParticipantsChanged = "participants_changed", + LocalMuteStateChanged = "local_mute_state_changed", } const CONF_ROOM = "me.robertlong.conf"; @@ -19,6 +25,14 @@ const SPEAKING_THRESHOLD = -80; const ACTIVE_SPEAKER_INTERVAL = 1000; const ACTIVE_SPEAKER_SAMPLES = 8; +export enum GroupCallParticipantEvent { + Speaking = "speaking", + VolumeChanged = "volume_changed", + MuteStateChanged = "mute_state_changed", + Datachannel = "datachannel", + CallReplaced = "call_replaced" +} + export class GroupCallParticipant extends EventEmitter { public feeds: CallFeed[] = []; public activeSpeaker: boolean; @@ -48,6 +62,8 @@ export class GroupCallParticipant extends EventEmitter { } public replaceCall(call: MatrixCall, sessionId: string) { + const oldCall = this.call; + if (this.call) { this.call.hangup(CallErrorCode.Replaced, false); this.call.removeListener(CallEvent.State, this.onCallStateChanged); @@ -70,6 +86,8 @@ export class GroupCallParticipant extends EventEmitter { this.call.on(CallEvent.Replaced, this.onCallReplaced); this.call.on(CallEvent.Hangup, this.onCallHangup); this.call.on(CallEvent.DataChannel, this.onCallDataChannel); + + this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); } public get usermediaFeed() { @@ -119,13 +137,6 @@ export class GroupCallParticipant extends EventEmitter { ) { call.setLocalVideoMuted(videoMuted); } - - this.groupCall.emit( - "debugstate", - this.member.userId, - this.call.callId, - state, - ); }; onCallFeedsChanged = () => { @@ -146,12 +157,10 @@ export class GroupCallParticipant extends EventEmitter { onCallReplaced = (newCall) => { // TODO: Should we always reuse the sessionId? this.replaceCall(newCall, this.sessionId); - this.groupCall.emit("call", newCall); - this.groupCall.emit("participants_changed"); }; onCallHangup = () => { - if (this.call.hangupReason === "replaced") { + if (this.call.hangupReason === CallErrorCode.Replaced) { return; } @@ -169,9 +178,10 @@ export class GroupCallParticipant extends EventEmitter { ) { this.groupCall.activeSpeaker = this.groupCall.participants[0]; this.groupCall.activeSpeaker.activeSpeaker = true; + this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); } - this.groupCall.emit("participants_changed"); + this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); }; addCallFeed(callFeed: CallFeed) { @@ -197,13 +207,13 @@ export class GroupCallParticipant extends EventEmitter { } onCallFeedSpeaking = (speaking: boolean) => { - this.emit("speaking"); + this.emit(GroupCallParticipantEvent.Speaking, speaking); }; onCallFeedVolumeChanged = (maxVolume: number) => { this.activeSpeakerSamples.shift(); this.activeSpeakerSamples.push(maxVolume); - this.emit("volume_changed", maxVolume); + this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); }; onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { @@ -213,12 +223,12 @@ export class GroupCallParticipant extends EventEmitter { ); } - this.emit("mute_state_changed", audioMuted, videoMuted); + this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); }; onCallDataChannel = (dataChannel: RTCDataChannel) => { this.dataChannel = dataChannel; - this.emit("datachannel"); + this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); }; } @@ -232,6 +242,7 @@ export class GroupCall extends EventEmitter { private speakerMap: Map = new Map(); private presenceLoopTimeout?: number; private activeSpeakerLoopTimeout: number; + private reEmitter: ReEmitter; constructor( private client: MatrixClient, @@ -243,6 +254,7 @@ export class GroupCall extends EventEmitter { super(); this.room = this.client.getRoom(roomId); + this.reEmitter = new ReEmitter(this); } async initLocalParticipant() { @@ -303,6 +315,7 @@ export class GroupCall extends EventEmitter { this.activeSpeaker = this.localParticipant; this.participants.push(this.localParticipant); + this.reEmitter.reEmit(this.localParticipant, Object.values(GroupCallParticipantEvent)); // Announce to the other room members that we have entered the room. // Continue doing so every PARTICIPANT_TIMEOUT ms @@ -323,8 +336,7 @@ export class GroupCall extends EventEmitter { this.client.on("RoomState.members", this.onRoomStateMembers); this.client.on("Call.incoming", this.onIncomingCall); - this.emit("entered"); - this.emit("participants_changed"); + this.emit(GroupCallEvent.Entered); this.onActiveSpeakerLoop(); } @@ -371,8 +383,7 @@ export class GroupCall extends EventEmitter { ); this.client.removeListener("Call.incoming", this.onIncomingCall); - this.emit("participants_changed"); - this.emit("left"); + this.emit(GroupCallEvent.Left); } isLocalVideoMuted() { @@ -412,8 +423,7 @@ export class GroupCall extends EventEmitter { } } - this.emit("participants_changed"); - this.emit("audio_mute_state_changed"); + this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); } setLocalVideoMuted(muted) { @@ -437,8 +447,7 @@ export class GroupCall extends EventEmitter { } } - this.emit("participants_changed"); - this.emit("video_mute_state_changed"); + this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); } public get localUsermediaFeed(): CallFeed { @@ -494,13 +503,6 @@ export class GroupCall extends EventEmitter { (memberStateContent[CONF_PARTICIPANT].expiresAt && memberStateContent[CONF_PARTICIPANT].expiresAt < now) ) { - this.emit( - "debugstate", - participant.member.userId, - null, - "inactive", - ); - if (participant.call) { // NOTE: This should remove the participant on the next tick // since matrix-js-sdk awaits a promise before firing user_hangup @@ -538,13 +540,15 @@ export class GroupCall extends EventEmitter { return; } - if (call.state !== "ringing") { + if (call.state !== CallState.Ringing) { logger.warn("Incoming call no longer in ringing state. Ignoring."); return; } const opponentMember = call.getOpponentMember(); + logger.log(`GroupCall: incomming call from: ${opponentMember.userId}`); + const memberStateEvent = this.room.currentState.getStateEvents( "m.room.member", opponentMember.userId, @@ -570,6 +574,7 @@ export class GroupCall extends EventEmitter { participant = existingParticipant; // This also fires the hangup event and triggers those side-effects existingParticipant.replaceCall(call, sessionId); + call.answer(); } else { participant = new GroupCallParticipant( this, @@ -578,24 +583,23 @@ export class GroupCall extends EventEmitter { call, ); this.participants.push(participant); + call.answer(); + this.reEmitter.reEmit(participant, Object.values(GroupCallParticipantEvent)); + this.emit(GroupCallEvent.ParticipantsChanged, this.participants); } - - call.answer(); - - this.emit("call", call); - this.emit("participants_changed"); - }; - - onRoomStateMembers = (_event, _state, member) => { - this.onMemberChanged(member); }; - onMemberChanged = (member) => { + onRoomStateMembers = (_event, _state, member: RoomMember) => { // The member events may be received for another room, which we will ignore. if (member.roomId !== this.room.roomId) { return; } + logger.log(`GroupCall member state changed: ${member.userId}`); + this.onMemberChanged(member); + }; + + onMemberChanged = (member: RoomMember) => { // Don't process your own member. const localUserId = this.client.getUserId(); @@ -626,7 +630,6 @@ export class GroupCall extends EventEmitter { const now = new Date().getTime(); if (expiresAt < now) { - this.emit("debugstate", member.userId, null, "inactive"); return; } @@ -639,7 +642,6 @@ export class GroupCall extends EventEmitter { if (participant) { if (participant.sessionId !== sessionId) { - this.emit("debugstate", member.userId, null, "inactive"); participant.call.hangup(CallErrorCode.Replaced, false); } else { return; @@ -649,12 +651,25 @@ export class GroupCall extends EventEmitter { // Only initiate a call with a user who has a userId that is lexicographically // less than your own. Otherwise, that user will call you. if (member.userId < localUserId) { - this.emit("debugstate", member.userId, null, "waiting for invite"); return; } const call = this.client.createCall(this.room.roomId, member.userId); + let callPromise; + + if (this.type === CallType.Video) { + callPromise = call.placeVideoCall(); + } else { + callPromise = call.placeVoiceCall(); + } + + callPromise.then(() => { + if (this.dataChannelsEnabled) { + call.createDataChannel("datachannel", this.dataChannelOptions); + } + }); + if (participant) { participant.replaceCall(call, sessionId); } else { @@ -669,25 +684,9 @@ export class GroupCall extends EventEmitter { // Does hiding a participant without a stream present a privacy problem because // a participant without a stream can still listen in on other user's streams? this.participants.push(participant); + this.reEmitter.reEmit(participant, Object.values(GroupCallParticipantEvent)); + this.emit(GroupCallEvent.ParticipantsChanged), this.participants; } - - let callPromise; - - if (this.type === CallType.Video) { - callPromise = call.placeVideoCall(); - } else { - callPromise = call.placeVoiceCall(); - } - - callPromise.then(() => { - if (this.dataChannelsEnabled) { - call.createDataChannel("datachannel", this.dataChannelOptions); - } - - this.emit("call", call); - }); - - this.emit("participants_changed"); }; onActiveSpeakerLoop = () => { @@ -715,7 +714,7 @@ export class GroupCall extends EventEmitter { this.activeSpeaker.activeSpeaker = false; nextActiveSpeaker.activeSpeaker = true; this.activeSpeaker = nextActiveSpeaker; - this.emit("participants_changed"); + this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } } @@ -731,11 +730,11 @@ export class GroupCall extends EventEmitter { // TODO: move this elsewhere or get rid of the retry logic. Do we need it? sendStateEventWithRetry( - roomId, - eventType, - content, - stateKey, - callback = undefined, + roomId: string, + eventType: string, + content: any, + stateKey?: string, + callback: Callback = undefined, maxAttempts = 5, ) { const sendStateEventWithRetry = async (attempt = 0) => { From 7021f70a66cdb46a0d919b7bcf6b20a00a007bf9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 9 Sep 2021 16:24:26 -0700 Subject: [PATCH 012/291] Move from constants to configureable public variables --- src/webrtc/groupCall.ts | 232 ++--------------------------- src/webrtc/groupCallParticipant.ts | 213 ++++++++++++++++++++++++++ 2 files changed, 226 insertions(+), 219 deletions(-) create mode 100644 src/webrtc/groupCallParticipant.ts diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3ff6f011cac..ce62768e46e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,14 +1,15 @@ import EventEmitter from "events"; -import { CallFeed, CallFeedEvent } from "./callFeed"; +import { CallFeed } from "./callFeed"; import { MatrixClient } from "../client"; import { randomString } from "../randomstring"; -import { CallErrorCode, CallEvent, CallState, CallType, MatrixCall } from "./call"; +import { CallErrorCode, CallState, CallType, MatrixCall } from "./call"; import { RoomMember } from "../models/room-member"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { Room } from "../models/room"; import { logger } from "../logger"; import { Callback } from "../client"; import { ReEmitter } from "../ReEmitter"; +import { GroupCallParticipant, GroupCallParticipantEvent } from "./groupCallParticipant"; export enum GroupCallEvent { Entered = "entered", @@ -20,217 +21,6 @@ export enum GroupCallEvent { const CONF_ROOM = "me.robertlong.conf"; const CONF_PARTICIPANT = "me.robertlong.conf.participant"; -const PARTICIPANT_TIMEOUT = 1000 * 15; -const SPEAKING_THRESHOLD = -80; -const ACTIVE_SPEAKER_INTERVAL = 1000; -const ACTIVE_SPEAKER_SAMPLES = 8; - -export enum GroupCallParticipantEvent { - Speaking = "speaking", - VolumeChanged = "volume_changed", - MuteStateChanged = "mute_state_changed", - Datachannel = "datachannel", - CallReplaced = "call_replaced" -} - -export class GroupCallParticipant extends EventEmitter { - public feeds: CallFeed[] = []; - public activeSpeaker: boolean; - public activeSpeakerSamples: number[]; - public dataChannel?: RTCDataChannel; - - constructor( - private groupCall: GroupCall, - public member: RoomMember, - // The session id is used to re-initiate calls if the user's participant - // session id has changed - public sessionId: string, - public call?: MatrixCall, - ) { - super(); - - this.activeSpeakerSamples = Array(ACTIVE_SPEAKER_SAMPLES).fill( - -Infinity, - ); - - if (this.call) { - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - } - } - - public replaceCall(call: MatrixCall, sessionId: string) { - const oldCall = this.call; - - if (this.call) { - this.call.hangup(CallErrorCode.Replaced, false); - this.call.removeListener(CallEvent.State, this.onCallStateChanged); - this.call.removeListener( - CallEvent.FeedsChanged, - this.onCallFeedsChanged, - ); - this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); - this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); - } - - this.call = call; - this.member = call.getOpponentMember(); - this.activeSpeaker = false; - this.sessionId = sessionId; - - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.DataChannel, this.onCallDataChannel); - - this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); - } - - public get usermediaFeed() { - return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get usermediaStream(): MediaStream { - return this.usermediaFeed?.stream; - } - - public isAudioMuted(): boolean { - const feed = this.usermediaFeed; - - if (!feed) { - return true; - } - - return feed.isAudioMuted(); - } - - public isVideoMuted(): boolean { - const feed = this.usermediaFeed; - - if (!feed) { - return true; - } - - return feed.isVideoMuted(); - } - - private onCallStateChanged = (state) => { - const call = this.call; - const audioMuted = this.groupCall.localParticipant.isAudioMuted(); - - if ( - call.localUsermediaStream && - call.isMicrophoneMuted() !== audioMuted - ) { - call.setMicrophoneMuted(audioMuted); - } - - const videoMuted = this.groupCall.localParticipant.isVideoMuted(); - - if ( - call.localUsermediaStream && - call.isLocalVideoMuted() !== videoMuted - ) { - call.setLocalVideoMuted(videoMuted); - } - }; - - onCallFeedsChanged = () => { - const oldFeeds = this.feeds; - const newFeeds = this.call.getRemoteFeeds(); - - this.feeds = []; - - for (const feed of newFeeds) { - if (oldFeeds.includes(feed)) { - continue; - } - - this.addCallFeed(feed); - } - }; - - onCallReplaced = (newCall) => { - // TODO: Should we always reuse the sessionId? - this.replaceCall(newCall, this.sessionId); - }; - - onCallHangup = () => { - if (this.call.hangupReason === CallErrorCode.Replaced) { - return; - } - - const participantIndex = this.groupCall.participants.indexOf(this); - - if (participantIndex === -1) { - return; - } - - this.groupCall.participants.splice(participantIndex, 1); - - if ( - this.groupCall.activeSpeaker === this && - this.groupCall.participants.length > 0 - ) { - this.groupCall.activeSpeaker = this.groupCall.participants[0]; - this.groupCall.activeSpeaker.activeSpeaker = true; - this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); - } - - this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); - }; - - addCallFeed(callFeed: CallFeed) { - if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { - callFeed.setSpeakingThreshold(SPEAKING_THRESHOLD); - callFeed.measureVolumeActivity(true); - callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - callFeed.on( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - callFeed.on( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - this.onCallFeedMuteStateChanged( - this.isAudioMuted(), - this.isVideoMuted(), - ); - } - - this.feeds.push(callFeed); - } - - onCallFeedSpeaking = (speaking: boolean) => { - this.emit(GroupCallParticipantEvent.Speaking, speaking); - }; - - onCallFeedVolumeChanged = (maxVolume: number) => { - this.activeSpeakerSamples.shift(); - this.activeSpeakerSamples.push(maxVolume); - this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); - }; - - onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { - if (audioMuted) { - this.activeSpeakerSamples = Array(ACTIVE_SPEAKER_SAMPLES).fill( - -Infinity, - ); - } - - this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); - }; - - onCallDataChannel = (dataChannel: RTCDataChannel) => { - this.dataChannel = dataChannel; - this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); - }; -} export class GroupCall extends EventEmitter { public entered = false; @@ -238,6 +28,10 @@ export class GroupCall extends EventEmitter { public localParticipant: GroupCallParticipant; public participants: GroupCallParticipant[] = []; public room: Room; + public activeSpeakerSampleCount = 8; + public activeSpeakerInterval = 1000; + public speakingThreshold = -80; + public participantTimeout = 1000 * 15; private speakerMap: Map = new Map(); private presenceLoopTimeout?: number; @@ -476,7 +270,7 @@ export class GroupCall extends EventEmitter { ...currentMemberState.getContent(), [CONF_PARTICIPANT]: { sessionId: this.localParticipant.sessionId, - expiresAt: new Date().getTime() + PARTICIPANT_TIMEOUT * 2, + expiresAt: new Date().getTime() + this.participantTimeout * 2, }, }, userId, @@ -513,7 +307,7 @@ export class GroupCall extends EventEmitter { this.presenceLoopTimeout = setTimeout( this.onPresenceLoop, - PARTICIPANT_TIMEOUT, + this.participantTimeout, ); }; @@ -698,10 +492,10 @@ export class GroupCall extends EventEmitter { for (let i = 0; i < participant.activeSpeakerSamples.length; i++) { const volume = participant.activeSpeakerSamples[i]; - total += Math.max(volume, SPEAKING_THRESHOLD); + total += Math.max(volume, this.speakingThreshold); } - const avg = total / ACTIVE_SPEAKER_SAMPLES; + const avg = total / this.activeSpeakerSampleCount; if (!topAvg || avg > topAvg) { topAvg = avg; @@ -709,7 +503,7 @@ export class GroupCall extends EventEmitter { } } - if (nextActiveSpeaker && topAvg > SPEAKING_THRESHOLD) { + if (nextActiveSpeaker && topAvg > this.speakingThreshold) { if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) { this.activeSpeaker.activeSpeaker = false; nextActiveSpeaker.activeSpeaker = true; @@ -720,7 +514,7 @@ export class GroupCall extends EventEmitter { this.activeSpeakerLoopTimeout = setTimeout( this.onActiveSpeakerLoop, - ACTIVE_SPEAKER_INTERVAL, + this.activeSpeakerInterval, ); }; diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts new file mode 100644 index 00000000000..8ab909bc9ab --- /dev/null +++ b/src/webrtc/groupCallParticipant.ts @@ -0,0 +1,213 @@ +import EventEmitter from "events"; +import { CallFeed, CallFeedEvent } from "./callFeed"; +import { CallErrorCode, CallEvent, MatrixCall } from "./call"; +import { RoomMember } from "../models/room-member"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { GroupCall, GroupCallEvent } from "./groupCall"; + +export enum GroupCallParticipantEvent { + Speaking = "speaking", + VolumeChanged = "volume_changed", + MuteStateChanged = "mute_state_changed", + Datachannel = "datachannel", + CallReplaced = "call_replaced" +} + +export class GroupCallParticipant extends EventEmitter { + public feeds: CallFeed[] = []; + public activeSpeaker: boolean; + public activeSpeakerSamples: number[]; + public dataChannel?: RTCDataChannel; + + constructor( + private groupCall: GroupCall, + public member: RoomMember, + // The session id is used to re-initiate calls if the user's participant + // session id has changed + public sessionId: string, + public call?: MatrixCall, + ) { + super(); + + this.activeSpeakerSamples = Array(groupCall.activeSpeakerSampleCount).fill( + -Infinity, + ); + + if (this.call) { + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + } + } + + public replaceCall(call: MatrixCall, sessionId: string) { + const oldCall = this.call; + + if (this.call) { + this.call.hangup(CallErrorCode.Replaced, false); + this.call.removeListener(CallEvent.State, this.onCallStateChanged); + this.call.removeListener( + CallEvent.FeedsChanged, + this.onCallFeedsChanged, + ); + this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); + this.call.removeListener(CallEvent.Hangup, this.onCallHangup); + this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); + } + + this.call = call; + this.member = call.getOpponentMember(); + this.activeSpeaker = false; + this.sessionId = sessionId; + + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + this.call.on(CallEvent.DataChannel, this.onCallDataChannel); + + this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); + } + + public get usermediaFeed() { + return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get usermediaStream(): MediaStream { + return this.usermediaFeed?.stream; + } + + public isAudioMuted(): boolean { + const feed = this.usermediaFeed; + + if (!feed) { + return true; + } + + return feed.isAudioMuted(); + } + + public isVideoMuted(): boolean { + const feed = this.usermediaFeed; + + if (!feed) { + return true; + } + + return feed.isVideoMuted(); + } + + private onCallStateChanged = (state) => { + const call = this.call; + const audioMuted = this.groupCall.localParticipant.isAudioMuted(); + + if ( + call.localUsermediaStream && + call.isMicrophoneMuted() !== audioMuted + ) { + call.setMicrophoneMuted(audioMuted); + } + + const videoMuted = this.groupCall.localParticipant.isVideoMuted(); + + if ( + call.localUsermediaStream && + call.isLocalVideoMuted() !== videoMuted + ) { + call.setLocalVideoMuted(videoMuted); + } + }; + + onCallFeedsChanged = () => { + const oldFeeds = this.feeds; + const newFeeds = this.call.getRemoteFeeds(); + + this.feeds = []; + + for (const feed of newFeeds) { + if (oldFeeds.includes(feed)) { + continue; + } + + this.addCallFeed(feed); + } + }; + + onCallReplaced = (newCall) => { + // TODO: Should we always reuse the sessionId? + this.replaceCall(newCall, this.sessionId); + }; + + onCallHangup = () => { + if (this.call.hangupReason === CallErrorCode.Replaced) { + return; + } + + const participantIndex = this.groupCall.participants.indexOf(this); + + if (participantIndex === -1) { + return; + } + + this.groupCall.participants.splice(participantIndex, 1); + + if ( + this.groupCall.activeSpeaker === this && + this.groupCall.participants.length > 0 + ) { + this.groupCall.activeSpeaker = this.groupCall.participants[0]; + this.groupCall.activeSpeaker.activeSpeaker = true; + this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); + } + + this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); + }; + + addCallFeed(callFeed: CallFeed) { + if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { + callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); + callFeed.measureVolumeActivity(true); + callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + callFeed.on( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + callFeed.on( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); + this.onCallFeedMuteStateChanged( + this.isAudioMuted(), + this.isVideoMuted(), + ); + } + + this.feeds.push(callFeed); + } + + onCallFeedSpeaking = (speaking: boolean) => { + this.emit(GroupCallParticipantEvent.Speaking, speaking); + }; + + onCallFeedVolumeChanged = (maxVolume: number) => { + this.activeSpeakerSamples.shift(); + this.activeSpeakerSamples.push(maxVolume); + this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); + }; + + onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { + if (audioMuted) { + this.activeSpeakerSamples = Array(this.groupCall.activeSpeakerSampleCount).fill( + -Infinity, + ); + } + + this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); + }; + + onCallDataChannel = (dataChannel: RTCDataChannel) => { + this.dataChannel = dataChannel; + this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); + }; +} From 07b2c5706494f9802a00007baf3076938c0b30e8 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 9 Sep 2021 16:40:08 -0700 Subject: [PATCH 013/291] Remove CallFeed export --- src/matrix.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/matrix.ts b/src/matrix.ts index d915c231050..3b7afb4b6cd 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -54,11 +54,6 @@ export { CallType, } from "./webrtc/call"; -// TODO: This export is temporary and is only used for the local call feed for conference calls -// Ideally conference calls will become a first-class concept and we will have a local call feed with -// a lifecycle that matches the conference call, not individual calls to members. -export { CallFeed } from "./webrtc/callFeed"; - // expose the underlying request object so different environments can use // different request libs (e.g. request or browser-request) let requestInstance; From 1dbb776e1230408343cb447f202c856ece2e8669 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 9 Sep 2021 17:07:18 -0700 Subject: [PATCH 014/291] Revert register types --- src/client.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index e8db288c604..7e39b44a4a5 100644 --- a/src/client.ts +++ b/src/client.ts @@ -6163,11 +6163,11 @@ export class MatrixClient extends EventEmitter { public register( username: string, password: string, - sessionId: string | null, + sessionId: string, auth: any, - bindThreepids?: any, - guestAccessToken?: string, - inhibitLogin?: boolean, + bindThreepids: any, + guestAccessToken: string, + inhibitLogin: boolean, callback?: Callback, ): Promise { // TODO: Types (many) // backwards compat From 2da14bd6e9cfb833943ae5b3137e7cd05beb40b7 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 10 Sep 2021 10:01:54 -0700 Subject: [PATCH 015/291] Fix call feed changed event handler --- src/webrtc/call.ts | 6 +-- src/webrtc/groupCallParticipant.ts | 62 +++++++++++++++++------------- 2 files changed, 38 insertions(+), 30 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8386afa3b46..47e106edf50 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -129,7 +129,7 @@ export enum CallEvent { AssertedIdentityChanged = 'asserted_identity_changed', - DataChannel = 'datachannel', + Datachannel = 'datachannel', LengthChanged = 'length_changed' } @@ -377,7 +377,7 @@ export class MatrixCall extends EventEmitter { public createDataChannel(label: string, options: RTCDataChannelInit) { logger.debug("createDataChannel"); const dataChannel = this.peerConn.createDataChannel(label, options); - this.emit(CallEvent.DataChannel, dataChannel); + this.emit(CallEvent.Datachannel, dataChannel); return dataChannel; } @@ -1543,7 +1543,7 @@ export class MatrixCall extends EventEmitter { }; private onDataChannel = (ev: RTCDataChannelEvent): void => { - this.emit(CallEvent.DataChannel, ev.channel); + this.emit(CallEvent.Datachannel, ev.channel); }; /** diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index 8ab909bc9ab..e0864980e2b 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -10,7 +10,8 @@ export enum GroupCallParticipantEvent { VolumeChanged = "volume_changed", MuteStateChanged = "mute_state_changed", Datachannel = "datachannel", - CallReplaced = "call_replaced" + CallReplaced = "call_replaced", + CallFeedsChanged = "call_feeds_changed" } export class GroupCallParticipant extends EventEmitter { @@ -38,6 +39,7 @@ export class GroupCallParticipant extends EventEmitter { this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); this.call.on(CallEvent.Replaced, this.onCallReplaced); this.call.on(CallEvent.Hangup, this.onCallHangup); + this.call.on(CallEvent.Datachannel, this.onCallDataChannel); } } @@ -53,7 +55,7 @@ export class GroupCallParticipant extends EventEmitter { ); this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); + this.call.removeListener(CallEvent.Datachannel, this.onCallDataChannel); } this.call = call; @@ -65,7 +67,7 @@ export class GroupCallParticipant extends EventEmitter { this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); this.call.on(CallEvent.Replaced, this.onCallReplaced); this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.DataChannel, this.onCallDataChannel); + this.call.on(CallEvent.Datachannel, this.onCallDataChannel); this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); } @@ -121,17 +123,15 @@ export class GroupCallParticipant extends EventEmitter { onCallFeedsChanged = () => { const oldFeeds = this.feeds; - const newFeeds = this.call.getRemoteFeeds(); + this.feeds = this.call.getRemoteFeeds(); - this.feeds = []; - - for (const feed of newFeeds) { - if (oldFeeds.includes(feed)) { - continue; + for (const feed of this.feeds) { + if (!oldFeeds.includes(feed) && feed.purpose === SDPStreamMetadataPurpose.Usermedia) { + this.addUserMediaFeed(feed); } - - this.addCallFeed(feed); } + + this.emit(GroupCallParticipantEvent.CallFeedsChanged); }; onCallReplaced = (newCall) => { @@ -164,26 +164,34 @@ export class GroupCallParticipant extends EventEmitter { this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); }; - addCallFeed(callFeed: CallFeed) { + addUserMediaFeed(callFeed: CallFeed) { + callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); + callFeed.measureVolumeActivity(true); + callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + callFeed.on( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + callFeed.on( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); + this.onCallFeedMuteStateChanged( + this.isAudioMuted(), + this.isVideoMuted(), + ); + } + + addCallFeed(callFeed: CallFeed, emitEvent?: boolean) { if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { - callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); - callFeed.measureVolumeActivity(true); - callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - callFeed.on( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - callFeed.on( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - this.onCallFeedMuteStateChanged( - this.isAudioMuted(), - this.isVideoMuted(), - ); + this.addUserMediaFeed(callFeed); } this.feeds.push(callFeed); + + if (emitEvent) { + this.emit(GroupCallParticipantEvent.CallFeedsChanged); + } } onCallFeedSpeaking = (speaking: boolean) => { From f18d8ead0869b8118eeed8311ae7285dc3e6298f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 10 Sep 2021 14:31:39 -0700 Subject: [PATCH 016/291] Fix usermedia feeds --- src/webrtc/groupCall.ts | 25 +++-- src/webrtc/groupCallParticipant.ts | 146 ++++++++++++++++------------- 2 files changed, 102 insertions(+), 69 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index ce62768e46e..cbe0adadfbd 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -4,12 +4,12 @@ import { MatrixClient } from "../client"; import { randomString } from "../randomstring"; import { CallErrorCode, CallState, CallType, MatrixCall } from "./call"; import { RoomMember } from "../models/room-member"; -import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { Room } from "../models/room"; import { logger } from "../logger"; import { Callback } from "../client"; import { ReEmitter } from "../ReEmitter"; import { GroupCallParticipant, GroupCallParticipantEvent } from "./groupCallParticipant"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; export enum GroupCallEvent { Entered = "entered", @@ -66,9 +66,11 @@ export class GroupCall extends EventEmitter { const userId = this.client.getUserId(); - const localCallFeed = new CallFeed( + const member = this.room.getMember(userId); + + const callFeed = new CallFeed( stream, - userId, + member.userId, SDPStreamMetadataPurpose.Usermedia, this.client, this.room.roomId, @@ -76,14 +78,13 @@ export class GroupCall extends EventEmitter { false, ); - const member = this.room.getMember(userId); - this.localParticipant = new GroupCallParticipant( this, member, randomString(16), ); - this.localParticipant.addCallFeed(localCallFeed); + + this.localParticipant.setLocalUsermediaFeed(callFeed); return this.localParticipant; } @@ -198,6 +199,12 @@ export class GroupCall extends EventEmitter { setMicrophoneMuted(muted) { if (this.localParticipant) { + const usermediaFeed = this.localParticipant.usermediaFeed; + + if (usermediaFeed) { + usermediaFeed.setAudioMuted(muted); + } + for (const { stream } of this.localParticipant.feeds) { for (const track of stream.getTracks()) { if (track.kind === "audio") { @@ -222,6 +229,12 @@ export class GroupCall extends EventEmitter { setLocalVideoMuted(muted) { if (this.localParticipant) { + const usermediaFeed = this.localParticipant.usermediaFeed; + + if (usermediaFeed) { + usermediaFeed.setVideoMuted(muted); + } + for (const { stream } of this.localParticipant.feeds) { for (const track of stream.getTracks()) { if (track.kind === "video") { diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index e0864980e2b..f551724eb96 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -15,10 +15,11 @@ export enum GroupCallParticipantEvent { } export class GroupCallParticipant extends EventEmitter { - public feeds: CallFeed[] = []; public activeSpeaker: boolean; public activeSpeakerSamples: number[]; public dataChannel?: RTCDataChannel; + private initializedUsermediaFeed?: CallFeed; + private localUsermediaFeed?: CallFeed; constructor( private groupCall: GroupCall, @@ -40,43 +41,35 @@ export class GroupCallParticipant extends EventEmitter { this.call.on(CallEvent.Replaced, this.onCallReplaced); this.call.on(CallEvent.Hangup, this.onCallHangup); this.call.on(CallEvent.Datachannel, this.onCallDataChannel); + + const usermediaFeed = this.usermediaFeed; + + if (usermediaFeed) { + this.initUserMediaFeed(usermediaFeed); + } } } - public replaceCall(call: MatrixCall, sessionId: string) { - const oldCall = this.call; + public setLocalUsermediaFeed(callFeed: CallFeed) { + this.localUsermediaFeed = callFeed; + this.initUserMediaFeed(callFeed); + } + public get feeds(): CallFeed[] { if (this.call) { - this.call.hangup(CallErrorCode.Replaced, false); - this.call.removeListener(CallEvent.State, this.onCallStateChanged); - this.call.removeListener( - CallEvent.FeedsChanged, - this.onCallFeedsChanged, - ); - this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); - this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.Datachannel, this.onCallDataChannel); + return this.call.getRemoteFeeds(); + } else if (this.localUsermediaFeed) { + return [this.localUsermediaFeed]; } - this.call = call; - this.member = call.getOpponentMember(); - this.activeSpeaker = false; - this.sessionId = sessionId; - - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.Datachannel, this.onCallDataChannel); - - this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); + return []; } - public get usermediaFeed() { + public get usermediaFeed(): CallFeed | undefined { return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); } - public get usermediaStream(): MediaStream { + public get usermediaStream(): MediaStream | undefined { return this.usermediaFeed?.stream; } @@ -100,7 +93,36 @@ export class GroupCallParticipant extends EventEmitter { return feed.isVideoMuted(); } - private onCallStateChanged = (state) => { + public replaceCall(call: MatrixCall, sessionId: string) { + const oldCall = this.call; + + if (this.call) { + this.call.hangup(CallErrorCode.Replaced, false); + this.call.removeListener(CallEvent.State, this.onCallStateChanged); + this.call.removeListener( + CallEvent.FeedsChanged, + this.onCallFeedsChanged, + ); + this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); + this.call.removeListener(CallEvent.Hangup, this.onCallHangup); + this.call.removeListener(CallEvent.Datachannel, this.onCallDataChannel); + } + + this.call = call; + this.member = call.getOpponentMember(); + this.activeSpeaker = false; + this.sessionId = sessionId; + + this.call.on(CallEvent.State, this.onCallStateChanged); + this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); + this.call.on(CallEvent.Replaced, this.onCallReplaced); + this.call.on(CallEvent.Hangup, this.onCallHangup); + this.call.on(CallEvent.Datachannel, this.onCallDataChannel); + + this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); + } + + private onCallStateChanged = () => { const call = this.call; const audioMuted = this.groupCall.localParticipant.isAudioMuted(); @@ -122,18 +144,46 @@ export class GroupCallParticipant extends EventEmitter { }; onCallFeedsChanged = () => { - const oldFeeds = this.feeds; - this.feeds = this.call.getRemoteFeeds(); - - for (const feed of this.feeds) { - if (!oldFeeds.includes(feed) && feed.purpose === SDPStreamMetadataPurpose.Usermedia) { - this.addUserMediaFeed(feed); + const nextUsermediaFeed = this.usermediaFeed; + + if (nextUsermediaFeed && nextUsermediaFeed !== this.initializedUsermediaFeed) { + if (this.initializedUsermediaFeed) { + this.initializedUsermediaFeed.removeListener(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + this.initializedUsermediaFeed.removeListener( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + this.initializedUsermediaFeed.removeListener( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); } + + this.initUserMediaFeed(nextUsermediaFeed); } this.emit(GroupCallParticipantEvent.CallFeedsChanged); }; + initUserMediaFeed(callFeed: CallFeed) { + this.initializedUsermediaFeed = callFeed; + callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); + callFeed.measureVolumeActivity(true); + callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + callFeed.on( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + callFeed.on( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); + this.onCallFeedMuteStateChanged( + this.isAudioMuted(), + this.isVideoMuted(), + ); + } + onCallReplaced = (newCall) => { // TODO: Should we always reuse the sessionId? this.replaceCall(newCall, this.sessionId); @@ -164,36 +214,6 @@ export class GroupCallParticipant extends EventEmitter { this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); }; - addUserMediaFeed(callFeed: CallFeed) { - callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); - callFeed.measureVolumeActivity(true); - callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - callFeed.on( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - callFeed.on( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - this.onCallFeedMuteStateChanged( - this.isAudioMuted(), - this.isVideoMuted(), - ); - } - - addCallFeed(callFeed: CallFeed, emitEvent?: boolean) { - if (callFeed.purpose === SDPStreamMetadataPurpose.Usermedia) { - this.addUserMediaFeed(callFeed); - } - - this.feeds.push(callFeed); - - if (emitEvent) { - this.emit(GroupCallParticipantEvent.CallFeedsChanged); - } - } - onCallFeedSpeaking = (speaking: boolean) => { this.emit(GroupCallParticipantEvent.Speaking, speaking); }; From eb2a47623f1306eb842da4b2f6f0ba300a39cc31 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 10 Sep 2021 15:58:44 -0700 Subject: [PATCH 017/291] Fix active speaker --- src/webrtc/groupCall.ts | 12 ++++-------- src/webrtc/groupCallParticipant.ts | 7 ++++--- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index cbe0adadfbd..3abb2dfb452 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -512,17 +512,13 @@ export class GroupCall extends EventEmitter { if (!topAvg || avg > topAvg) { topAvg = avg; - nextActiveSpeaker = participant.member; + nextActiveSpeaker = participant; } } - if (nextActiveSpeaker && topAvg > this.speakingThreshold) { - if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker) { - this.activeSpeaker.activeSpeaker = false; - nextActiveSpeaker.activeSpeaker = true; - this.activeSpeaker = nextActiveSpeaker; - this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); - } + if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg > this.speakingThreshold) { + this.activeSpeaker = nextActiveSpeaker; + this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } this.activeSpeakerLoopTimeout = setTimeout( diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index f551724eb96..fee3eb64852 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -15,7 +15,6 @@ export enum GroupCallParticipantEvent { } export class GroupCallParticipant extends EventEmitter { - public activeSpeaker: boolean; public activeSpeakerSamples: number[]; public dataChannel?: RTCDataChannel; private initializedUsermediaFeed?: CallFeed; @@ -93,6 +92,10 @@ export class GroupCallParticipant extends EventEmitter { return feed.isVideoMuted(); } + public isActiveSpeaker(): boolean { + return this.groupCall.activeSpeaker === this; + } + public replaceCall(call: MatrixCall, sessionId: string) { const oldCall = this.call; @@ -110,7 +113,6 @@ export class GroupCallParticipant extends EventEmitter { this.call = call; this.member = call.getOpponentMember(); - this.activeSpeaker = false; this.sessionId = sessionId; this.call.on(CallEvent.State, this.onCallStateChanged); @@ -207,7 +209,6 @@ export class GroupCallParticipant extends EventEmitter { this.groupCall.participants.length > 0 ) { this.groupCall.activeSpeaker = this.groupCall.participants[0]; - this.groupCall.activeSpeaker.activeSpeaker = true; this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); } From fb3ca90bc9441dfa781dffe4cb432f1dc9dbe659 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 10 Sep 2021 16:06:26 -0700 Subject: [PATCH 018/291] Fix private method signatures --- src/webrtc/callFeed.ts | 6 +++++- src/webrtc/groupCall.ts | 28 ++++++++++++++-------------- src/webrtc/groupCallParticipant.ts | 16 ++++++++-------- 3 files changed, 27 insertions(+), 23 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 29a8a76687e..56a7beffee1 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -35,7 +35,7 @@ export class CallFeed extends EventEmitter { private analyser: AnalyserNode; private frequencyBinCount: Float32Array; private speakingThreshold = SPEAKING_THRESHOLD; - public speaking = false; + private speaking = false; private volumeLooperTimeout: number; constructor( @@ -110,6 +110,10 @@ export class CallFeed extends EventEmitter { return this.stream.getVideoTracks().length === 0 || this.videoMuted; } + public isSpeaking(): boolean { + return this.speaking; + } + /** * Replaces the current MediaStream with a new one. * This method should be only used by MatrixCall. diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3abb2dfb452..5c0f8b6b171 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -51,7 +51,7 @@ export class GroupCall extends EventEmitter { this.reEmitter = new ReEmitter(this); } - async initLocalParticipant() { + public async initLocalParticipant() { if (this.localParticipant) { return this.localParticipant; } @@ -89,7 +89,7 @@ export class GroupCall extends EventEmitter { return this.localParticipant; } - async enter() { + public async enter() { if (!this.localParticipant) { await this.initLocalParticipant(); } @@ -135,7 +135,7 @@ export class GroupCall extends EventEmitter { this.onActiveSpeakerLoop(); } - leave() { + public leave() { this.localParticipant = null; this.client.stopLocalMediaStream(); @@ -181,7 +181,7 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.Left); } - isLocalVideoMuted() { + public isLocalVideoMuted() { if (this.localParticipant) { return this.localParticipant.isVideoMuted(); } @@ -189,7 +189,7 @@ export class GroupCall extends EventEmitter { return true; } - isMicrophoneMuted() { + public isMicrophoneMuted() { if (this.localParticipant) { return this.localParticipant.isAudioMuted(); } @@ -197,7 +197,7 @@ export class GroupCall extends EventEmitter { return true; } - setMicrophoneMuted(muted) { + public setMicrophoneMuted(muted) { if (this.localParticipant) { const usermediaFeed = this.localParticipant.usermediaFeed; @@ -227,7 +227,7 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); } - setLocalVideoMuted(muted) { + public setLocalVideoMuted(muted) { if (this.localParticipant) { const usermediaFeed = this.localParticipant.usermediaFeed; @@ -269,7 +269,7 @@ export class GroupCall extends EventEmitter { * Call presence */ - onPresenceLoop = () => { + private onPresenceLoop = () => { const userId = this.client.getUserId(); const currentMemberState = this.room.currentState.getStateEvents( "m.room.member", @@ -333,7 +333,7 @@ export class GroupCall extends EventEmitter { * as they are observed by the RoomState.members event. */ - processInitialCalls() { + private processInitialCalls() { const calls = this.client.callEventHandler.calls.values(); for (const call of calls) { @@ -341,7 +341,7 @@ export class GroupCall extends EventEmitter { } } - onIncomingCall = (call: MatrixCall) => { + private onIncomingCall = (call: MatrixCall) => { // The incoming calls may be for another room, which we will ignore. if (call.roomId !== this.room.roomId) { return; @@ -396,7 +396,7 @@ export class GroupCall extends EventEmitter { } }; - onRoomStateMembers = (_event, _state, member: RoomMember) => { + private onRoomStateMembers = (_event, _state, member: RoomMember) => { // The member events may be received for another room, which we will ignore. if (member.roomId !== this.room.roomId) { return; @@ -406,7 +406,7 @@ export class GroupCall extends EventEmitter { this.onMemberChanged(member); }; - onMemberChanged = (member: RoomMember) => { + private onMemberChanged = (member: RoomMember) => { // Don't process your own member. const localUserId = this.client.getUserId(); @@ -496,7 +496,7 @@ export class GroupCall extends EventEmitter { } }; - onActiveSpeakerLoop = () => { + private onActiveSpeakerLoop = () => { let topAvg; let nextActiveSpeaker; @@ -532,7 +532,7 @@ export class GroupCall extends EventEmitter { */ // TODO: move this elsewhere or get rid of the retry logic. Do we need it? - sendStateEventWithRetry( + private sendStateEventWithRetry( roomId: string, eventType: string, content: any, diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index fee3eb64852..f0ecc255c6f 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -145,7 +145,7 @@ export class GroupCallParticipant extends EventEmitter { } }; - onCallFeedsChanged = () => { + private onCallFeedsChanged = () => { const nextUsermediaFeed = this.usermediaFeed; if (nextUsermediaFeed && nextUsermediaFeed !== this.initializedUsermediaFeed) { @@ -167,7 +167,7 @@ export class GroupCallParticipant extends EventEmitter { this.emit(GroupCallParticipantEvent.CallFeedsChanged); }; - initUserMediaFeed(callFeed: CallFeed) { + private initUserMediaFeed(callFeed: CallFeed) { this.initializedUsermediaFeed = callFeed; callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); callFeed.measureVolumeActivity(true); @@ -186,12 +186,12 @@ export class GroupCallParticipant extends EventEmitter { ); } - onCallReplaced = (newCall) => { + private onCallReplaced = (newCall) => { // TODO: Should we always reuse the sessionId? this.replaceCall(newCall, this.sessionId); }; - onCallHangup = () => { + private onCallHangup = () => { if (this.call.hangupReason === CallErrorCode.Replaced) { return; } @@ -215,17 +215,17 @@ export class GroupCallParticipant extends EventEmitter { this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); }; - onCallFeedSpeaking = (speaking: boolean) => { + private onCallFeedSpeaking = (speaking: boolean) => { this.emit(GroupCallParticipantEvent.Speaking, speaking); }; - onCallFeedVolumeChanged = (maxVolume: number) => { + private onCallFeedVolumeChanged = (maxVolume: number) => { this.activeSpeakerSamples.shift(); this.activeSpeakerSamples.push(maxVolume); this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); }; - onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { + private onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { if (audioMuted) { this.activeSpeakerSamples = Array(this.groupCall.activeSpeakerSampleCount).fill( -Infinity, @@ -235,7 +235,7 @@ export class GroupCallParticipant extends EventEmitter { this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); }; - onCallDataChannel = (dataChannel: RTCDataChannel) => { + private onCallDataChannel = (dataChannel: RTCDataChannel) => { this.dataChannel = dataChannel; this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); }; From 2d7e07f4ed3362538d04935d9bc8f1472e59e8b8 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 15 Sep 2021 12:45:42 -0700 Subject: [PATCH 019/291] Update to use latest datachannel / clone media stream PRs --- src/webrtc/groupCall.ts | 10 ++-------- src/webrtc/groupCallParticipant.ts | 6 +++--- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 5c0f8b6b171..c366ed55cf0 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -56,13 +56,7 @@ export class GroupCall extends EventEmitter { return this.localParticipant; } - let stream; - - if (this.type === CallType.Video) { - stream = await this.client.getLocalVideoStream(); - } else { - stream = await this.client.getLocalAudioStream(); - } + const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === CallType.Video); const userId = this.client.getUserId(); @@ -137,7 +131,7 @@ export class GroupCall extends EventEmitter { public leave() { this.localParticipant = null; - this.client.stopLocalMediaStream(); + this.client.getMediaHandler().stopAllStreams(); if (!this.entered) { return; diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index f0ecc255c6f..a09f18db21e 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -39,7 +39,7 @@ export class GroupCallParticipant extends EventEmitter { this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); this.call.on(CallEvent.Replaced, this.onCallReplaced); this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.Datachannel, this.onCallDataChannel); + this.call.on(CallEvent.DataChannel, this.onCallDataChannel); const usermediaFeed = this.usermediaFeed; @@ -108,7 +108,7 @@ export class GroupCallParticipant extends EventEmitter { ); this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.Datachannel, this.onCallDataChannel); + this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); } this.call = call; @@ -119,7 +119,7 @@ export class GroupCallParticipant extends EventEmitter { this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); this.call.on(CallEvent.Replaced, this.onCallReplaced); this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.Datachannel, this.onCallDataChannel); + this.call.on(CallEvent.DataChannel, this.onCallDataChannel); this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); } From 8b1c1736591ffeef7c9048e2287026c5ffcae730 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 16 Sep 2021 16:35:41 -0700 Subject: [PATCH 020/291] Avoid changing member on replaced call --- src/webrtc/groupCallParticipant.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index a09f18db21e..327934fb2d9 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -112,7 +112,6 @@ export class GroupCallParticipant extends EventEmitter { } this.call = call; - this.member = call.getOpponentMember(); this.sessionId = sessionId; this.call.on(CallEvent.State, this.onCallStateChanged); From aef5d73de4677a457deefedd4a0cf93867e2cee6 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 20 Sep 2021 17:30:49 -0700 Subject: [PATCH 021/291] Fix emitting participants_changed event --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c366ed55cf0..f22d90577ad 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -486,7 +486,7 @@ export class GroupCall extends EventEmitter { // a participant without a stream can still listen in on other user's streams? this.participants.push(participant); this.reEmitter.reEmit(participant, Object.values(GroupCallParticipantEvent)); - this.emit(GroupCallEvent.ParticipantsChanged), this.participants; + this.emit(GroupCallEvent.ParticipantsChanged, this.participants); } }; From 18986cb33a1b75cf93defc11d39bc729ce4fc843 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 20 Sep 2021 17:31:02 -0700 Subject: [PATCH 022/291] Fix typo --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index f22d90577ad..e4c611b81bc 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -348,7 +348,7 @@ export class GroupCall extends EventEmitter { const opponentMember = call.getOpponentMember(); - logger.log(`GroupCall: incomming call from: ${opponentMember.userId}`); + logger.log(`GroupCall: incoming call from: ${opponentMember.userId}`); const memberStateEvent = this.room.currentState.getStateEvents( "m.room.member", From 151b54ed651be54f776c6dd25621d241e7eb97c3 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 21 Sep 2021 14:24:37 -0700 Subject: [PATCH 023/291] Clean up GroupCallParticipant listeners on remove --- src/webrtc/groupCallParticipant.ts | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index 327934fb2d9..2afb7c328e6 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -203,6 +203,29 @@ export class GroupCallParticipant extends EventEmitter { this.groupCall.participants.splice(participantIndex, 1); + if (this.call) { + this.call.removeListener(CallEvent.State, this.onCallStateChanged); + this.call.removeListener( + CallEvent.FeedsChanged, + this.onCallFeedsChanged, + ); + this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); + this.call.removeListener(CallEvent.Hangup, this.onCallHangup); + this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); + } + + if (this.initializedUsermediaFeed) { + this.initializedUsermediaFeed.removeListener(CallFeedEvent.Speaking, this.onCallFeedSpeaking); + this.initializedUsermediaFeed.removeListener( + CallFeedEvent.VolumeChanged, + this.onCallFeedVolumeChanged, + ); + this.initializedUsermediaFeed.removeListener( + CallFeedEvent.MuteStateChanged, + this.onCallFeedMuteStateChanged, + ); + } + if ( this.groupCall.activeSpeaker === this && this.groupCall.participants.length > 0 From bbeea51a3676e4a1873494ba6520fe0e5914202f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 21 Sep 2021 14:24:51 -0700 Subject: [PATCH 024/291] Add callType to room state event --- .babelrc | 2 +- src/webrtc/groupCall.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.babelrc b/.babelrc index 6f4f54a93f1..ca46890fdbb 100644 --- a/.babelrc +++ b/.babelrc @@ -5,7 +5,7 @@ "targets": { "node": 10 }, - "modules": "commonjs" + "modules": false }], "@babel/preset-typescript" ], diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e4c611b81bc..981acc3c624 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -97,7 +97,7 @@ export class GroupCall extends EventEmitter { this.sendStateEventWithRetry( this.room.roomId, CONF_ROOM, - { active: true }, + { active: true, callType: this.type }, "", ); } From 6e07c9e900e19f9effb87c6d4785c2836f426d2b Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 21 Sep 2021 14:48:22 -0700 Subject: [PATCH 025/291] Clean up group call event listeners properly on hangup --- src/webrtc/groupCall.ts | 15 +++++++++++---- src/webrtc/groupCallParticipant.ts | 6 +++++- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 981acc3c624..7e7bf859499 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -153,14 +153,21 @@ export class GroupCall extends EventEmitter { userId, ); - for (const participant of this.participants) { + // Clean up participant event listeners and hangup calls + // Reverse iteration because participant.remove() removes the participant from the participants array. + for (let i = this.participants.length - 1; i >= 0; i--) { + const participant = this.participants[i]; + + participant.remove(); + + // Hangup is async, so we call remove which removes all the call event listeners + // that reference this group call if (participant.call) { participant.call.hangup(CallErrorCode.UserHangup, false); } } this.entered = false; - this.participants = []; this.activeSpeaker = null; this.speakerMap.clear(); clearTimeout(this.presenceLoopTimeout); @@ -304,9 +311,9 @@ export class GroupCall extends EventEmitter { (memberStateContent[CONF_PARTICIPANT].expiresAt && memberStateContent[CONF_PARTICIPANT].expiresAt < now) ) { + participant.remove(); + if (participant.call) { - // NOTE: This should remove the participant on the next tick - // since matrix-js-sdk awaits a promise before firing user_hangup participant.call.hangup(CallErrorCode.UserHangup, false); } } diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts index 2afb7c328e6..4c225ae161e 100644 --- a/src/webrtc/groupCallParticipant.ts +++ b/src/webrtc/groupCallParticipant.ts @@ -195,6 +195,10 @@ export class GroupCallParticipant extends EventEmitter { return; } + this.remove(); + }; + + public remove() { const participantIndex = this.groupCall.participants.indexOf(this); if (participantIndex === -1) { @@ -235,7 +239,7 @@ export class GroupCallParticipant extends EventEmitter { } this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); - }; + } private onCallFeedSpeaking = (speaking: boolean) => { this.emit(GroupCallParticipantEvent.Speaking, speaking); From 7f189b0abd9b2b0a9637a78a16e46d8ab6cbec04 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 21 Sep 2021 17:10:45 -0700 Subject: [PATCH 026/291] Add endCall --- src/webrtc/groupCall.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e4c611b81bc..c6601301ce8 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -175,6 +175,15 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.Left); } + public async endCall() { + await this.sendStateEventWithRetry( + this.room.roomId, + CONF_ROOM, + { active: false }, + "", + ); + } + public isLocalVideoMuted() { if (this.localParticipant) { return this.localParticipant.isVideoMuted(); From 5895ce32fafb382e692f32def69aba5d0198aee9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 21 Sep 2021 21:00:47 -0700 Subject: [PATCH 027/291] Revert unintended babelrc edit --- .babelrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.babelrc b/.babelrc index ca46890fdbb..6f4f54a93f1 100644 --- a/.babelrc +++ b/.babelrc @@ -5,7 +5,7 @@ "targets": { "node": 10 }, - "modules": false + "modules": "commonjs" }], "@babel/preset-typescript" ], From 61a80a11c9af1c2d2d3d73ff64c93ac255bb043a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 22 Sep 2021 12:49:21 +0200 Subject: [PATCH 028/291] Export CONF_ROOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 78adffe528f..80a3db8e938 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -19,7 +19,7 @@ export enum GroupCallEvent { LocalMuteStateChanged = "local_mute_state_changed", } -const CONF_ROOM = "me.robertlong.conf"; +export const CONF_ROOM = "me.robertlong.conf"; const CONF_PARTICIPANT = "me.robertlong.conf.participant"; export class GroupCall extends EventEmitter { From 9996afed033ac912051e7a60d62fbf4eff1ea5fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 22 Sep 2021 12:49:33 +0200 Subject: [PATCH 029/291] Throw with no room MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 80a3db8e938..a38c4e2fd56 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -48,6 +48,9 @@ export class GroupCall extends EventEmitter { super(); this.room = this.client.getRoom(roomId); + if (!this.room) { + throw new Error("Can't find the room"); + } this.reEmitter = new ReEmitter(this); } From 55ef57ead81af6f9a0df6a07dcc8f67ebc90ff4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 22 Sep 2021 14:59:55 +0200 Subject: [PATCH 030/291] Add GroupCallEventHandler MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 5 +++ src/webrtc/groupCallEventHandler.ts | 61 +++++++++++++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 src/webrtc/groupCallEventHandler.ts diff --git a/src/client.ts b/src/client.ts index 2d8f86e0c1b..6909ae94baa 100644 --- a/src/client.ts +++ b/src/client.ts @@ -146,6 +146,7 @@ import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; import { GroupCall } from "./webrtc/groupCall"; import { MediaHandler } from "./webrtc/mediaHandler"; +import { GroupCallEventHandler } from "./webrtc/groupCallEventHandler"; export type Store = IStore; export type SessionStore = WebStorageSessionStore; @@ -696,6 +697,7 @@ export class MatrixClient extends EventEmitter { public crypto: Crypto; // XXX: Intended private, used in code. public cryptoCallbacks: ICryptoCallbacks; // XXX: Intended private, used in code. public callEventHandler: CallEventHandler; // XXX: Intended private, used in code. + private groupCallEventHandler: GroupCallEventHandler; public supportsCallTransfer = false; // XXX: Intended private, used in code. public forceTURN = false; // XXX: Intended private, used in code. public iceCandidatePoolSize = 0; // XXX: Intended private, used in code. @@ -815,6 +817,7 @@ export class MatrixClient extends EventEmitter { const call = createNewMatrixCall(this, undefined, undefined); if (call) { this.callEventHandler = new CallEventHandler(this); + this.groupCallEventHandler = new GroupCallEventHandler(this); this.canSupportVoip = true; // Start listening for calls after the initial sync is done // We do not need to backfill the call event buffer @@ -1005,6 +1008,7 @@ export class MatrixClient extends EventEmitter { this.peekSync?.stopPeeking(); this.callEventHandler?.stop(); + this.groupCallEventHandler?.stop(); this.callEventHandler = null; global.clearInterval(this.checkTurnServersIntervalID); @@ -5591,6 +5595,7 @@ export class MatrixClient extends EventEmitter { private startCallEventHandler = (): void => { if (this.isInitialSyncComplete()) { this.callEventHandler.start(); + this.groupCallEventHandler.start(); this.off("sync", this.startCallEventHandler); } }; diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts new file mode 100644 index 00000000000..d0e54ff4326 --- /dev/null +++ b/src/webrtc/groupCallEventHandler.ts @@ -0,0 +1,61 @@ +/* +Copyright 2021 Šimon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixEvent } from '../models/event'; +import { MatrixClient } from '../client'; +import { CONF_ROOM, GroupCall } from "./groupCall"; +import { RoomState } from "../models/room-state"; +import { CallType } from "./call"; + +export class GroupCallEventHandler { + private groupCalls = new Map(); // roomId -> GroupCall + + constructor(private client: MatrixClient) { } + + public start(): void { + this.client.on("RoomState.events", this.onRoomStateChanged); + } + + public stop(): void { + this.client.removeListener("RoomState.events", this.onRoomStateChanged); + } + + private onRoomStateChanged = (_event: MatrixEvent, state: RoomState): void => { + const groupCall = this.groupCalls.get(state.roomId); + const confEvents = state.getStateEvents(CONF_ROOM); + let content; + if (confEvents.length > 0) { + content = confEvents[0].getContent(); + } + + if (groupCall && !content?.active) { + groupCall.leave(); + this.groupCalls.delete(state.roomId); + } else if (!groupCall && content?.active) { + let callType: CallType; + + if (content.callType === "voice") { + callType = CallType.Voice; + } else { + callType = CallType.Video; + } + + const groupCall = this.client.createGroupCall(state.roomId, callType); + this.groupCalls.set(state.roomId, groupCall); + this.client.emit("GroupCall.incoming", groupCall); + } + }; +} From 9c7cb3cbea0f77a7ee8c9c77ba5cddab7dcc7108 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 22 Sep 2021 15:03:48 -0700 Subject: [PATCH 031/291] Handle more edge cases around creating/ending group calls --- src/client.ts | 52 ++++++++++++++++++++++++----- src/webrtc/groupCall.ts | 37 +++++++++----------- src/webrtc/groupCallEventHandler.ts | 35 ++++++++++++++++--- 3 files changed, 90 insertions(+), 34 deletions(-) diff --git a/src/client.ts b/src/client.ts index 6909ae94baa..a7ec0e4532d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -144,7 +144,7 @@ import { IHierarchyRoom, ISpaceSummaryEvent, ISpaceSummaryRoom } from "./@types/ import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; -import { GroupCall } from "./webrtc/groupCall"; +import { CONF_ROOM, GroupCall, IGroupCallDataChannelOptions } from "./webrtc/groupCall"; import { MediaHandler } from "./webrtc/mediaHandler"; import { GroupCallEventHandler } from "./webrtc/groupCallEventHandler"; @@ -697,7 +697,7 @@ export class MatrixClient extends EventEmitter { public crypto: Crypto; // XXX: Intended private, used in code. public cryptoCallbacks: ICryptoCallbacks; // XXX: Intended private, used in code. public callEventHandler: CallEventHandler; // XXX: Intended private, used in code. - private groupCallEventHandler: GroupCallEventHandler; + public groupCallEventHandler: GroupCallEventHandler; public supportsCallTransfer = false; // XXX: Intended private, used in code. public forceTURN = false; // XXX: Intended private, used in code. public iceCandidatePoolSize = 0; // XXX: Intended private, used in code. @@ -1290,18 +1290,54 @@ export class MatrixClient extends EventEmitter { } /** - * Creates a new group call. + * Creates a new group call and sends the associated state event + * to alert other members that the room now has a group call. * * @param {string} roomId The room the call is to be placed in. - * @return {GroupCall} the call or null if the browser doesn't support calling. + * @return {GroupCall} */ - public createGroupCall( + public async createGroupCall( roomId: string, type: CallType, dataChannelsEnabled?: boolean, - dataChannelOptions?: RTCDataChannelInit, - ): GroupCall { - return new GroupCall(this, roomId, type, dataChannelsEnabled, dataChannelOptions); + dataChannelOptions?: IGroupCallDataChannelOptions, + ): Promise { + if (this.getGroupCallForRoom(roomId)) { + throw new Error(`${roomId} already has an existing group call`); + } + + const room = this.getRoom(roomId); + + if (!room) { + throw new Error(`Cannot find room ${roomId}`); + } + + const groupCall = new GroupCall(this, room, type, dataChannelsEnabled, dataChannelOptions); + this.groupCallEventHandler.groupCalls.set(roomId, groupCall); + + const activeConf = room.currentState + .getStateEvents(CONF_ROOM, "") + ?.getContent()?.active; + + if (!activeConf) { + await this.sendStateEvent( + room.roomId, + CONF_ROOM, + { active: true, callType: type, dataChannelsEnabled, dataChannelOptions }, + "", + ); + } + + return groupCall; + } + + /** + * Get an existing group call for the provided room. + * @param roomId + * @returns {GroupCall} The group call or null if it doesn't already exist. + */ + public getGroupCallForRoom(roomId: string): GroupCall | null { + return this.groupCallEventHandler.groupCalls.get(roomId) || null; } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a38c4e2fd56..370aa371572 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -22,12 +22,18 @@ export enum GroupCallEvent { export const CONF_ROOM = "me.robertlong.conf"; const CONF_PARTICIPANT = "me.robertlong.conf.participant"; +export interface IGroupCallDataChannelOptions { + ordered: boolean; + maxPacketLifeTime: number; + maxRetransmits: number; + protocol: string; +} + export class GroupCall extends EventEmitter { public entered = false; public activeSpeaker: GroupCallParticipant; public localParticipant: GroupCallParticipant; public participants: GroupCallParticipant[] = []; - public room: Room; public activeSpeakerSampleCount = 8; public activeSpeakerInterval = 1000; public speakingThreshold = -80; @@ -40,17 +46,12 @@ export class GroupCall extends EventEmitter { constructor( private client: MatrixClient, - roomId: string, + public room: Room, public type: CallType, private dataChannelsEnabled?: boolean, - private dataChannelOptions?: RTCDataChannelInit, + private dataChannelOptions?: IGroupCallDataChannelOptions, ) { super(); - - this.room = this.client.getRoom(roomId); - if (!this.room) { - throw new Error("Can't find the room"); - } this.reEmitter = new ReEmitter(this); } @@ -91,20 +92,6 @@ export class GroupCall extends EventEmitter { await this.initLocalParticipant(); } - // Ensure that this room is marked as a conference room so clients can react appropriately - const activeConf = this.room.currentState - .getStateEvents(CONF_ROOM, "") - ?.getContent()?.active; - - if (!activeConf) { - this.sendStateEventWithRetry( - this.room.roomId, - CONF_ROOM, - { active: true, callType: this.type }, - "", - ); - } - this.activeSpeaker = this.localParticipant; this.participants.push(this.localParticipant); this.reEmitter.reEmit(this.localParticipant, Object.values(GroupCallParticipantEvent)); @@ -186,6 +173,12 @@ export class GroupCall extends EventEmitter { } public async endCall() { + this.leave(); + + this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); + + this.client.emit("GroupCall.ended", this); + await this.sendStateEventWithRetry( this.room.roomId, CONF_ROOM, diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index d0e54ff4326..4226072138e 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -16,12 +16,13 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { MatrixClient } from '../client'; -import { CONF_ROOM, GroupCall } from "./groupCall"; +import { CONF_ROOM, GroupCall, IGroupCallDataChannelOptions } from "./groupCall"; import { RoomState } from "../models/room-state"; import { CallType } from "./call"; +import { logger } from '../logger'; export class GroupCallEventHandler { - private groupCalls = new Map(); // roomId -> GroupCall + public groupCalls = new Map(); // roomId -> GroupCall constructor(private client: MatrixClient) { } @@ -41,9 +42,14 @@ export class GroupCallEventHandler { content = confEvents[0].getContent(); } - if (groupCall && !content?.active) { + if (groupCall && content?.type !== groupCall.type) { + // TODO: Handle the callType changing when the room state changes + logger.warn(`The group call type changed for room: ${ + state.roomId}. Changing the group call type is currently unsupported.`); + } if (groupCall && !content?.active) { groupCall.leave(); this.groupCalls.delete(state.roomId); + this.client.emit("GroupCall.ended", groupCall); } else if (!groupCall && content?.active) { let callType: CallType; @@ -53,7 +59,28 @@ export class GroupCallEventHandler { callType = CallType.Video; } - const groupCall = this.client.createGroupCall(state.roomId, callType); + const room = this.client.getRoom(state.roomId); + + if (!room) { + logger.error(`Couldn't find room ${state.roomId} for GroupCall`); + return; + } + + let dataChannelOptions: IGroupCallDataChannelOptions | undefined; + + if (content?.dataChannelsEnabled && content?.dataChannelOptions) { + // Pull out just the dataChannelOptions we want to support. + const { ordered, maxPacketLifeTime, maxRetransmits, protocol } = content.dataChannelOptions; + dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol }; + } + + const groupCall = new GroupCall( + this.client, + room, + callType, + content?.dataChannelsEnabled, + dataChannelOptions, + ); this.groupCalls.set(state.roomId, groupCall); this.client.emit("GroupCall.incoming", groupCall); } From ac70dcfc9110cc2905eba005f792b8b75c11bbab Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 23 Sep 2021 16:23:32 -0700 Subject: [PATCH 032/291] Expose call feed getters on call --- src/webrtc/call.ts | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ced87180391..4add21221bd 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -420,10 +420,26 @@ export class MatrixCall extends EventEmitter { return this.localUsermediaFeed?.stream; } - private get localScreensharingStream(): MediaStream { + public get localScreensharingStream(): MediaStream { return this.localScreensharingFeed?.stream; } + public get remoteUsermediaFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); + } + + public get remoteScreensharingFeed(): CallFeed { + return this.getRemoteFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + } + + public get remoteUsermediaStream(): MediaStream { + return this.remoteUsermediaFeed?.stream; + } + + public get remoteScreensharingStream(): MediaStream { + return this.remoteScreensharingFeed?.stream; + } + private getFeedByStreamId(streamId: string): CallFeed { return this.getFeeds().find((feed) => feed.stream.id === streamId); } From 89bda6c2e5e6ab90c871c333254a4c17f3950ba3 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 24 Sep 2021 12:39:43 -0700 Subject: [PATCH 033/291] Move from groupCallsParticipants to calls --- src/webrtc/call.ts | 2 +- src/webrtc/groupCall.ts | 566 ++++++++++++++++++----------- src/webrtc/groupCallParticipant.ts | 268 -------------- 3 files changed, 347 insertions(+), 489 deletions(-) delete mode 100644 src/webrtc/groupCallParticipant.ts diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4add21221bd..c06dddb646e 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2054,7 +2054,7 @@ export class MatrixCall extends EventEmitter { } } -function setTracksEnabled(tracks: Array, enabled: boolean): void { +export function setTracksEnabled(tracks: Array, enabled: boolean): void { for (let i = 0; i < tracks.length; i++) { tracks[i].enabled = enabled; } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 370aa371572..70d95ab9199 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,21 +1,20 @@ import EventEmitter from "events"; -import { CallFeed } from "./callFeed"; +import { CallFeed, CallFeedEvent } from "./callFeed"; import { MatrixClient } from "../client"; import { randomString } from "../randomstring"; -import { CallErrorCode, CallState, CallType, MatrixCall } from "./call"; +import { CallErrorCode, CallEvent, CallState, CallType, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; -import { Callback } from "../client"; import { ReEmitter } from "../ReEmitter"; -import { GroupCallParticipant, GroupCallParticipantEvent } from "./groupCallParticipant"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; export enum GroupCallEvent { Entered = "entered", Left = "left", ActiveSpeakerChanged = "active_speaker_changed", - ParticipantsChanged = "participants_changed", + CallsChanged = "calls_changed", + UserMediaFeedsChanged = "user_media_feeds_changed", LocalMuteStateChanged = "local_mute_state_changed", } @@ -29,19 +28,37 @@ export interface IGroupCallDataChannelOptions { protocol: string; } +interface IUserMediaFeedHandlers { + onCallFeedVolumeChanged: (maxVolume: number) => void; + onCallFeedMuteStateChanged: (audioMuted: boolean) => void; +} + +interface ICallHandlers { + onCallFeedsChanged: (feeds: CallFeed[]) => void; + onCallStateChanged: (state: CallState, oldState: CallState) => void; + onCallHangup: (call: MatrixCall) => void; +} + export class GroupCall extends EventEmitter { - public entered = false; - public activeSpeaker: GroupCallParticipant; - public localParticipant: GroupCallParticipant; - public participants: GroupCallParticipant[] = []; + // Config public activeSpeakerSampleCount = 8; public activeSpeakerInterval = 1000; public speakingThreshold = -80; public participantTimeout = 1000 * 15; - private speakerMap: Map = new Map(); + public entered = false; + public activeSpeaker: string; // userId + public localCallFeed: CallFeed; + public calls: MatrixCall[] = []; + public userMediaFeeds: CallFeed[] = []; + + private userMediaFeedHandlers: Map = new Map(); + private callHandlers: Map = new Map(); + private sessionIds: Map = new Map(); // userId -> sessionId + private activeSpeakerSamples: Map; + private presenceLoopTimeout?: number; - private activeSpeakerLoopTimeout: number; + private activeSpeakerLoopTimeout?: number; private reEmitter: ReEmitter; constructor( @@ -55,20 +72,18 @@ export class GroupCall extends EventEmitter { this.reEmitter = new ReEmitter(this); } - public async initLocalParticipant() { - if (this.localParticipant) { - return this.localParticipant; + public async initLocalCallFeed(): Promise { + if (this.localCallFeed) { + return this.localCallFeed; } const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === CallType.Video); const userId = this.client.getUserId(); - const member = this.room.getMember(userId); - const callFeed = new CallFeed( stream, - member.userId, + userId, SDPStreamMetadataPurpose.Usermedia, this.client, this.room.roomId, @@ -76,25 +91,22 @@ export class GroupCall extends EventEmitter { false, ); - this.localParticipant = new GroupCallParticipant( - this, - member, - randomString(16), - ); - - this.localParticipant.setLocalUsermediaFeed(callFeed); + this.sessionIds.set(userId, randomString(16)); + this.activeSpeakerSamples.set(userId, Array(this.activeSpeakerSampleCount).fill( + -Infinity, + )); + this.localCallFeed = callFeed; + this.addUserMediaFeed(callFeed); - return this.localParticipant; + return callFeed; } public async enter() { - if (!this.localParticipant) { - await this.initLocalParticipant(); + if (!this.localCallFeed) { + await this.initLocalCallFeed(); } - this.activeSpeaker = this.localParticipant; - this.participants.push(this.localParticipant); - this.reEmitter.reEmit(this.localParticipant, Object.values(GroupCallParticipantEvent)); + this.activeSpeaker = this.client.getUserId(); // Announce to the other room members that we have entered the room. // Continue doing so every PARTICIPANT_TIMEOUT ms @@ -120,7 +132,11 @@ export class GroupCall extends EventEmitter { } public leave() { - this.localParticipant = null; + if (this.localCallFeed) { + this.removeUserMediaFeed(this.localCallFeed); + this.localCallFeed = null; + } + this.client.getMediaHandler().stopAllStreams(); if (!this.entered) { @@ -133,7 +149,7 @@ export class GroupCall extends EventEmitter { userId, ); - this.sendStateEventWithRetry( + this.client.sendStateEvent( this.room.roomId, "m.room.member", { @@ -143,23 +159,13 @@ export class GroupCall extends EventEmitter { userId, ); - // Clean up participant event listeners and hangup calls - // Reverse iteration because participant.remove() removes the participant from the participants array. - for (let i = this.participants.length - 1; i >= 0; i--) { - const participant = this.participants[i]; - - participant.remove(); - - // Hangup is async, so we call remove which removes all the call event listeners - // that reference this group call - if (participant.call) { - participant.call.hangup(CallErrorCode.UserHangup, false); - } + while (this.calls.length > 0) { + const call = this.calls.pop(); + this.removeCall(call, CallErrorCode.UserHangup); } this.entered = false; this.activeSpeaker = null; - this.speakerMap.clear(); clearTimeout(this.presenceLoopTimeout); clearTimeout(this.activeSpeakerLoopTimeout); @@ -179,7 +185,7 @@ export class GroupCall extends EventEmitter { this.client.emit("GroupCall.ended", this); - await this.sendStateEventWithRetry( + await this.client.sendStateEvent( this.room.roomId, CONF_ROOM, { active: false }, @@ -187,124 +193,86 @@ export class GroupCall extends EventEmitter { ); } + /** + * Local Usermedia + */ + public isLocalVideoMuted() { - if (this.localParticipant) { - return this.localParticipant.isVideoMuted(); + if (this.localCallFeed) { + return this.localCallFeed.isVideoMuted(); } return true; } public isMicrophoneMuted() { - if (this.localParticipant) { - return this.localParticipant.isAudioMuted(); + if (this.localCallFeed) { + return this.localCallFeed.isAudioMuted(); } return true; } public setMicrophoneMuted(muted) { - if (this.localParticipant) { - const usermediaFeed = this.localParticipant.usermediaFeed; - - if (usermediaFeed) { - usermediaFeed.setAudioMuted(muted); - } - - for (const { stream } of this.localParticipant.feeds) { - for (const track of stream.getTracks()) { - if (track.kind === "audio") { - track.enabled = !muted; - } - } - } + if (this.localCallFeed) { + this.localCallFeed.setAudioMuted(muted); + setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); } - for (const { call } of this.participants) { - if ( - call && - call.localUsermediaStream && - call.isMicrophoneMuted() !== muted - ) { - call.setMicrophoneMuted(muted); - } + for (const call of this.calls) { + call.setMicrophoneMuted(muted); } this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); } public setLocalVideoMuted(muted) { - if (this.localParticipant) { - const usermediaFeed = this.localParticipant.usermediaFeed; - - if (usermediaFeed) { - usermediaFeed.setVideoMuted(muted); - } - - for (const { stream } of this.localParticipant.feeds) { - for (const track of stream.getTracks()) { - if (track.kind === "video") { - track.enabled = !muted; - } - } - } + if (this.localCallFeed) { + this.localCallFeed.setVideoMuted(muted); + setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); } - for (const { call } of this.participants) { - if ( - call && - call.localUsermediaStream && - call.isLocalVideoMuted() !== muted - ) { - call.setLocalVideoMuted(muted); - } + for (const call of this.calls) { + call.setLocalVideoMuted(muted); } this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); } - public get localUsermediaFeed(): CallFeed { - return this.localParticipant?.usermediaFeed; - } - - public get localUsermediaStream(): MediaStream { - return this.localParticipant?.usermediaStream; - } - /** * Call presence */ private onPresenceLoop = () => { - const userId = this.client.getUserId(); + const localUserId = this.client.getUserId(); const currentMemberState = this.room.currentState.getStateEvents( "m.room.member", - userId, + localUserId, ); - this.sendStateEventWithRetry( + this.client.sendStateEvent( this.room.roomId, "m.room.member", { ...currentMemberState.getContent(), [CONF_PARTICIPANT]: { - sessionId: this.localParticipant.sessionId, + sessionId: this.sessionIds.get(localUserId), expiresAt: new Date().getTime() + this.participantTimeout * 2, }, }, - userId, + localUserId, ); const now = new Date().getTime(); - for (const participant of this.participants) { - if (participant === this.localParticipant) { - continue; - } + // Iterate backwards so that we can remove items + for (let i = this.calls.length - 1; i >= 0; i--) { + const call = this.calls[i]; + const opponentUserId = call.getOpponentMember().userId; const memberStateEvent = this.room.currentState.getStateEvents( "m.room.member", - participant.member.userId, + opponentUserId, ); const memberStateContent = memberStateEvent.getContent(); @@ -316,11 +284,7 @@ export class GroupCall extends EventEmitter { (memberStateContent[CONF_PARTICIPANT].expiresAt && memberStateContent[CONF_PARTICIPANT].expiresAt < now) ) { - participant.remove(); - - if (participant.call) { - participant.call.hangup(CallErrorCode.UserHangup, false); - } + this.removeCall(call, CallErrorCode.UserHangup); } } @@ -347,59 +311,46 @@ export class GroupCall extends EventEmitter { } } - private onIncomingCall = (call: MatrixCall) => { + private onIncomingCall = (newCall: MatrixCall) => { // The incoming calls may be for another room, which we will ignore. - if (call.roomId !== this.room.roomId) { + if (newCall.roomId !== this.room.roomId) { return; } - if (call.state !== CallState.Ringing) { + if (newCall.state !== CallState.Ringing) { logger.warn("Incoming call no longer in ringing state. Ignoring."); return; } - const opponentMember = call.getOpponentMember(); + const opponentMemberId = newCall.getOpponentMember().userId; - logger.log(`GroupCall: incoming call from: ${opponentMember.userId}`); + logger.log(`GroupCall: incoming call from: ${opponentMemberId}`); const memberStateEvent = this.room.currentState.getStateEvents( "m.room.member", - opponentMember.userId, + opponentMemberId, ); const memberStateContent = memberStateEvent.getContent(); if (!memberStateContent || !memberStateContent[CONF_PARTICIPANT]) { - call.reject(); + newCall.reject(); return; } const { sessionId } = memberStateContent[CONF_PARTICIPANT]; + this.sessionIds.set(opponentMemberId, sessionId); - // Check if the user calling has an existing participant and use this call instead. - const existingParticipant = this.participants.find( - (participant) => participant.member.userId === opponentMember.userId, - ); - - let participant; + const existingCall = this.getCallByUserId(opponentMemberId); - if (existingParticipant) { - participant = existingParticipant; - // This also fires the hangup event and triggers those side-effects - existingParticipant.replaceCall(call, sessionId); - call.answer(); + // Check if the user calling has an existing call and use this call instead. + if (existingCall) { + this.replaceCall(existingCall, newCall, sessionId); } else { - participant = new GroupCallParticipant( - this, - opponentMember, - sessionId, - call, - ); - this.participants.push(participant); - call.answer(); - this.reEmitter.reEmit(participant, Object.values(GroupCallParticipantEvent)); - this.emit(GroupCallEvent.ParticipantsChanged, this.participants); + this.addCall(newCall, sessionId); } + + newCall.answer(); }; private onRoomStateMembers = (_event, _state, member: RoomMember) => { @@ -446,19 +397,13 @@ export class GroupCall extends EventEmitter { return; } - // If there is an existing participant for this member check the session id. + // If there is an existing call for this member check the session id. // If the session id changed then we can hang up the old call and start a new one. // Otherwise, ignore the member change event because we already have an active participant. - let participant = this.participants.find( - (p) => p.member.userId === member.userId, - ); + const existingCall = this.getCallByUserId(member.userId); - if (participant) { - if (participant.sessionId !== sessionId) { - participant.call.hangup(CallErrorCode.Replaced, false); - } else { - return; - } + if (existingCall && this.sessionIds.get(member.userId) === sessionId) { + return; } // Only initiate a call with a user who has a userId that is lexicographically @@ -467,50 +412,267 @@ export class GroupCall extends EventEmitter { return; } - const call = this.client.createCall(this.room.roomId, member.userId); - - let callPromise; + const newCall = this.client.createCall(this.room.roomId, member.userId); - if (this.type === CallType.Video) { - callPromise = call.placeVideoCall(); - } else { - callPromise = call.placeVoiceCall(); - } + // TODO: Move to call.placeCall() + const callPromise = this.type === CallType.Video ? newCall.placeVideoCall() : newCall.placeVoiceCall(); callPromise.then(() => { if (this.dataChannelsEnabled) { - call.createDataChannel("datachannel", this.dataChannelOptions); + newCall.createDataChannel("datachannel", this.dataChannelOptions); } }); - if (participant) { - participant.replaceCall(call, sessionId); + if (existingCall) { + this.replaceCall(existingCall, newCall, sessionId); } else { - participant = new GroupCallParticipant( - this, - member, - sessionId, - call, + this.addCall(newCall, sessionId); + } + }; + + /** + * Call Event Handlers + */ + + public getCallByUserId(userId: string): MatrixCall { + return this.calls.find((call) => call.getOpponentMember().userId === userId); + } + + private addCall(call: MatrixCall, sessionId: string) { + this.calls.push(call); + this.initCall(call, sessionId); + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + + private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, sessionId: string) { + const existingCallIndex = this.calls.indexOf(existingCall); + + if (existingCallIndex === -1) { + throw new Error("Couldn't find call to replace"); + } + + this.calls.splice(existingCallIndex, 1, replacementCall); + + this.disposeCall(existingCall, CallErrorCode.Replaced); + this.initCall(replacementCall, sessionId); + + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + + private removeCall(call: MatrixCall, hangupReason: CallErrorCode) { + this.disposeCall(call, hangupReason); + + const callIndex = this.calls.indexOf(call); + + if (callIndex === -1) { + throw new Error("Couldn't find call to remove"); + } + + this.calls.splice(callIndex, 1); + + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + + private initCall(call: MatrixCall, sessionId: string) { + const opponentMemberId = call.getOpponentMember().userId; + + const onCallFeedsChanged = (feeds: CallFeed[]) => this.onCallFeedsChanged(call, feeds); + const onCallStateChanged = + (state: CallState, oldState: CallState) => this.onCallStateChanged(call, state, oldState); + const onCallHangup = this.onCallHangup; + + this.callHandlers.set(opponentMemberId, { + onCallFeedsChanged, + onCallStateChanged, + onCallHangup, + }); + + call.on(CallEvent.FeedsChanged, onCallFeedsChanged); + call.on(CallEvent.State, onCallStateChanged); + call.on(CallEvent.Hangup, onCallHangup); + + this.activeSpeakerSamples.set(opponentMemberId, Array(this.activeSpeakerSampleCount).fill( + -Infinity, + )); + this.sessionIds.set(opponentMemberId, sessionId); + this.reEmitter.reEmit(call, Object.values(CallEvent)); + } + + private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { + const opponentMemberId = call.getOpponentMember().userId; + + const { + onCallFeedsChanged, + onCallStateChanged, + onCallHangup, + } = this.callHandlers.get(opponentMemberId); + + call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); + call.removeListener(CallEvent.State, onCallStateChanged); + call.removeListener(CallEvent.Hangup, onCallHangup); + + this.callHandlers.delete(opponentMemberId); + + if (call.state !== CallState.Ended) { + call.hangup(hangupReason, false); + } + + const usermediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + + if (usermediaFeed) { + this.removeUserMediaFeed(usermediaFeed); + } + + this.activeSpeakerSamples.delete(opponentMemberId); + this.sessionIds.delete(opponentMemberId); + } + + private onCallFeedsChanged = (call: MatrixCall, feeds: CallFeed[]) => { + const opponentMemberId = call.getOpponentMember().userId; + const currentUserMediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + + let newUserMediaFeed: CallFeed; + + for (const feed of feeds) { + if (feed.purpose === SDPStreamMetadataPurpose.Usermedia && feed !== currentUserMediaFeed) { + newUserMediaFeed = feed; + } + } + + if (!currentUserMediaFeed && newUserMediaFeed) { + this.addUserMediaFeed(newUserMediaFeed); + } else if (currentUserMediaFeed && newUserMediaFeed) { + this.replaceUserMediaFeed(currentUserMediaFeed, newUserMediaFeed); + } else if (currentUserMediaFeed && !newUserMediaFeed) { + this.removeUserMediaFeed(currentUserMediaFeed); + } + }; + + private onCallStateChanged = (call: MatrixCall, _state: CallState, _oldState: CallState) => { + const audioMuted = this.localCallFeed.isAudioMuted(); + + if ( + call.localUsermediaStream && + call.isMicrophoneMuted() !== audioMuted + ) { + call.setMicrophoneMuted(audioMuted); + } + + const videoMuted = this.localCallFeed.isVideoMuted(); + + if ( + call.localUsermediaStream && + call.isLocalVideoMuted() !== videoMuted + ) { + call.setLocalVideoMuted(videoMuted); + } + }; + + private onCallHangup = (call: MatrixCall) => { + if (call.hangupReason === CallErrorCode.Replaced) { + return; + } + + this.removeCall(call, call.hangupReason as CallErrorCode); + }; + + /** + * UserMedia CallFeed Event Handlers + */ + + public getUserMediaFeedByUserId(userId: string) { + return this.userMediaFeeds.find((feed) => feed.userId === userId); + } + + private addUserMediaFeed(callFeed: CallFeed) { + this.userMediaFeeds.push(callFeed); + this.initUserMediaFeed(callFeed); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + } + + private replaceUserMediaFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { + const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === existingFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find user media feed to replace"); + } + + this.userMediaFeeds.splice(feedIndex, 1, replacementFeed); + + this.disposeUserMediaFeed(existingFeed); + this.initUserMediaFeed(replacementFeed); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + } + + private removeUserMediaFeed(callFeed: CallFeed) { + const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === callFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find user media feed to remove"); + } + + this.userMediaFeeds.splice(feedIndex, 1); + + this.disposeUserMediaFeed(callFeed); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + + if ( + this.activeSpeaker === callFeed.userId && + this.userMediaFeeds.length > 0 + ) { + this.activeSpeaker = this.userMediaFeeds[0].userId; + this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); + } + } + + private initUserMediaFeed(callFeed: CallFeed) { + callFeed.setSpeakingThreshold(this.speakingThreshold); + callFeed.measureVolumeActivity(true); + + const onCallFeedVolumeChanged = (maxVolume: number) => this.onCallFeedVolumeChanged(callFeed, maxVolume); + const onCallFeedMuteStateChanged = + (audioMuted: boolean) => this.onCallFeedMuteStateChanged(callFeed, audioMuted); + + this.userMediaFeedHandlers.set(callFeed.userId, { + onCallFeedVolumeChanged, + onCallFeedMuteStateChanged, + }); + + callFeed.on(CallFeedEvent.VolumeChanged, onCallFeedVolumeChanged); + callFeed.on(CallFeedEvent.MuteStateChanged, onCallFeedMuteStateChanged); + } + + private disposeUserMediaFeed(callFeed: CallFeed) { + const { onCallFeedVolumeChanged, onCallFeedMuteStateChanged } = this.userMediaFeedHandlers.get(callFeed.userId); + callFeed.removeListener(CallFeedEvent.VolumeChanged, onCallFeedVolumeChanged); + callFeed.removeListener(CallFeedEvent.MuteStateChanged, onCallFeedMuteStateChanged); + this.userMediaFeedHandlers.delete(callFeed.userId); + callFeed.dispose(); + } + + private onCallFeedVolumeChanged = (callFeed: CallFeed, maxVolume: number) => { + const activeSpeakerSamples = this.activeSpeakerSamples.get(callFeed.userId); + activeSpeakerSamples.shift(); + activeSpeakerSamples.push(maxVolume); + }; + + private onCallFeedMuteStateChanged = (callFeed: CallFeed, audioMuted: boolean) => { + if (audioMuted) { + this.activeSpeakerSamples.get(callFeed.userId).fill( + -Infinity, ); - // TODO: Should we wait until the call has been answered to push the participant? - // Or do we hide the participant until their stream is live? - // Does hiding a participant without a stream present a privacy problem because - // a participant without a stream can still listen in on other user's streams? - this.participants.push(participant); - this.reEmitter.reEmit(participant, Object.values(GroupCallParticipantEvent)); - this.emit(GroupCallEvent.ParticipantsChanged, this.participants); } }; private onActiveSpeakerLoop = () => { - let topAvg; - let nextActiveSpeaker; + let topAvg: number; + let nextActiveSpeaker: string; - for (const participant of this.participants) { + for (const [userId, samples] of this.activeSpeakerSamples) { let total = 0; - for (let i = 0; i < participant.activeSpeakerSamples.length; i++) { - const volume = participant.activeSpeakerSamples[i]; + for (let i = 0; i < samples.length; i++) { + const volume = samples[i]; total += Math.max(volume, this.speakingThreshold); } @@ -518,7 +680,7 @@ export class GroupCall extends EventEmitter { if (!topAvg || avg > topAvg) { topAvg = avg; - nextActiveSpeaker = participant; + nextActiveSpeaker = userId; } } @@ -532,40 +694,4 @@ export class GroupCall extends EventEmitter { this.activeSpeakerInterval, ); }; - - /** - * Utils - */ - - // TODO: move this elsewhere or get rid of the retry logic. Do we need it? - private sendStateEventWithRetry( - roomId: string, - eventType: string, - content: any, - stateKey?: string, - callback: Callback = undefined, - maxAttempts = 5, - ) { - const sendStateEventWithRetry = async (attempt = 0) => { - try { - return await this.client.sendStateEvent( - roomId, - eventType, - content, - stateKey, - callback, - ); - } catch (error) { - if (attempt >= maxAttempts) { - throw error; - } - - await new Promise((resolve) => setTimeout(resolve, 5)); - - return sendStateEventWithRetry(attempt + 1); - } - }; - - return sendStateEventWithRetry(); - } } diff --git a/src/webrtc/groupCallParticipant.ts b/src/webrtc/groupCallParticipant.ts deleted file mode 100644 index 4c225ae161e..00000000000 --- a/src/webrtc/groupCallParticipant.ts +++ /dev/null @@ -1,268 +0,0 @@ -import EventEmitter from "events"; -import { CallFeed, CallFeedEvent } from "./callFeed"; -import { CallErrorCode, CallEvent, MatrixCall } from "./call"; -import { RoomMember } from "../models/room-member"; -import { SDPStreamMetadataPurpose } from "./callEventTypes"; -import { GroupCall, GroupCallEvent } from "./groupCall"; - -export enum GroupCallParticipantEvent { - Speaking = "speaking", - VolumeChanged = "volume_changed", - MuteStateChanged = "mute_state_changed", - Datachannel = "datachannel", - CallReplaced = "call_replaced", - CallFeedsChanged = "call_feeds_changed" -} - -export class GroupCallParticipant extends EventEmitter { - public activeSpeakerSamples: number[]; - public dataChannel?: RTCDataChannel; - private initializedUsermediaFeed?: CallFeed; - private localUsermediaFeed?: CallFeed; - - constructor( - private groupCall: GroupCall, - public member: RoomMember, - // The session id is used to re-initiate calls if the user's participant - // session id has changed - public sessionId: string, - public call?: MatrixCall, - ) { - super(); - - this.activeSpeakerSamples = Array(groupCall.activeSpeakerSampleCount).fill( - -Infinity, - ); - - if (this.call) { - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.DataChannel, this.onCallDataChannel); - - const usermediaFeed = this.usermediaFeed; - - if (usermediaFeed) { - this.initUserMediaFeed(usermediaFeed); - } - } - } - - public setLocalUsermediaFeed(callFeed: CallFeed) { - this.localUsermediaFeed = callFeed; - this.initUserMediaFeed(callFeed); - } - - public get feeds(): CallFeed[] { - if (this.call) { - return this.call.getRemoteFeeds(); - } else if (this.localUsermediaFeed) { - return [this.localUsermediaFeed]; - } - - return []; - } - - public get usermediaFeed(): CallFeed | undefined { - return this.feeds.find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia); - } - - public get usermediaStream(): MediaStream | undefined { - return this.usermediaFeed?.stream; - } - - public isAudioMuted(): boolean { - const feed = this.usermediaFeed; - - if (!feed) { - return true; - } - - return feed.isAudioMuted(); - } - - public isVideoMuted(): boolean { - const feed = this.usermediaFeed; - - if (!feed) { - return true; - } - - return feed.isVideoMuted(); - } - - public isActiveSpeaker(): boolean { - return this.groupCall.activeSpeaker === this; - } - - public replaceCall(call: MatrixCall, sessionId: string) { - const oldCall = this.call; - - if (this.call) { - this.call.hangup(CallErrorCode.Replaced, false); - this.call.removeListener(CallEvent.State, this.onCallStateChanged); - this.call.removeListener( - CallEvent.FeedsChanged, - this.onCallFeedsChanged, - ); - this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); - this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); - } - - this.call = call; - this.sessionId = sessionId; - - this.call.on(CallEvent.State, this.onCallStateChanged); - this.call.on(CallEvent.FeedsChanged, this.onCallFeedsChanged); - this.call.on(CallEvent.Replaced, this.onCallReplaced); - this.call.on(CallEvent.Hangup, this.onCallHangup); - this.call.on(CallEvent.DataChannel, this.onCallDataChannel); - - this.groupCall.emit(GroupCallParticipantEvent.CallReplaced, this, oldCall, call); - } - - private onCallStateChanged = () => { - const call = this.call; - const audioMuted = this.groupCall.localParticipant.isAudioMuted(); - - if ( - call.localUsermediaStream && - call.isMicrophoneMuted() !== audioMuted - ) { - call.setMicrophoneMuted(audioMuted); - } - - const videoMuted = this.groupCall.localParticipant.isVideoMuted(); - - if ( - call.localUsermediaStream && - call.isLocalVideoMuted() !== videoMuted - ) { - call.setLocalVideoMuted(videoMuted); - } - }; - - private onCallFeedsChanged = () => { - const nextUsermediaFeed = this.usermediaFeed; - - if (nextUsermediaFeed && nextUsermediaFeed !== this.initializedUsermediaFeed) { - if (this.initializedUsermediaFeed) { - this.initializedUsermediaFeed.removeListener(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - this.initializedUsermediaFeed.removeListener( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - this.initializedUsermediaFeed.removeListener( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - } - - this.initUserMediaFeed(nextUsermediaFeed); - } - - this.emit(GroupCallParticipantEvent.CallFeedsChanged); - }; - - private initUserMediaFeed(callFeed: CallFeed) { - this.initializedUsermediaFeed = callFeed; - callFeed.setSpeakingThreshold(this.groupCall.speakingThreshold); - callFeed.measureVolumeActivity(true); - callFeed.on(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - callFeed.on( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - callFeed.on( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - this.onCallFeedMuteStateChanged( - this.isAudioMuted(), - this.isVideoMuted(), - ); - } - - private onCallReplaced = (newCall) => { - // TODO: Should we always reuse the sessionId? - this.replaceCall(newCall, this.sessionId); - }; - - private onCallHangup = () => { - if (this.call.hangupReason === CallErrorCode.Replaced) { - return; - } - - this.remove(); - }; - - public remove() { - const participantIndex = this.groupCall.participants.indexOf(this); - - if (participantIndex === -1) { - return; - } - - this.groupCall.participants.splice(participantIndex, 1); - - if (this.call) { - this.call.removeListener(CallEvent.State, this.onCallStateChanged); - this.call.removeListener( - CallEvent.FeedsChanged, - this.onCallFeedsChanged, - ); - this.call.removeListener(CallEvent.Replaced, this.onCallReplaced); - this.call.removeListener(CallEvent.Hangup, this.onCallHangup); - this.call.removeListener(CallEvent.DataChannel, this.onCallDataChannel); - } - - if (this.initializedUsermediaFeed) { - this.initializedUsermediaFeed.removeListener(CallFeedEvent.Speaking, this.onCallFeedSpeaking); - this.initializedUsermediaFeed.removeListener( - CallFeedEvent.VolumeChanged, - this.onCallFeedVolumeChanged, - ); - this.initializedUsermediaFeed.removeListener( - CallFeedEvent.MuteStateChanged, - this.onCallFeedMuteStateChanged, - ); - } - - if ( - this.groupCall.activeSpeaker === this && - this.groupCall.participants.length > 0 - ) { - this.groupCall.activeSpeaker = this.groupCall.participants[0]; - this.groupCall.emit(GroupCallEvent.ActiveSpeakerChanged, this.groupCall.activeSpeaker); - } - - this.groupCall.emit(GroupCallEvent.ParticipantsChanged, this.groupCall.participants); - } - - private onCallFeedSpeaking = (speaking: boolean) => { - this.emit(GroupCallParticipantEvent.Speaking, speaking); - }; - - private onCallFeedVolumeChanged = (maxVolume: number) => { - this.activeSpeakerSamples.shift(); - this.activeSpeakerSamples.push(maxVolume); - this.emit(GroupCallParticipantEvent.VolumeChanged, maxVolume); - }; - - private onCallFeedMuteStateChanged = (audioMuted: boolean, videoMuted: boolean) => { - if (audioMuted) { - this.activeSpeakerSamples = Array(this.groupCall.activeSpeakerSampleCount).fill( - -Infinity, - ); - } - - this.emit(GroupCallParticipantEvent.MuteStateChanged, audioMuted, videoMuted); - }; - - private onCallDataChannel = (dataChannel: RTCDataChannel) => { - this.dataChannel = dataChannel; - this.emit(GroupCallParticipantEvent.Datachannel, dataChannel); - }; -} From 454da84f6ede37c9403eab082d1a6ae689792141 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 24 Sep 2021 13:29:23 -0700 Subject: [PATCH 034/291] Initialize activeSpeakerSamples --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 70d95ab9199..65c12caaa8d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -55,7 +55,7 @@ export class GroupCall extends EventEmitter { private userMediaFeedHandlers: Map = new Map(); private callHandlers: Map = new Map(); private sessionIds: Map = new Map(); // userId -> sessionId - private activeSpeakerSamples: Map; + private activeSpeakerSamples: Map = new Map(); private presenceLoopTimeout?: number; private activeSpeakerLoopTimeout?: number; From 96f48929acda3202a325a5692f4d95317e94f928 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 24 Sep 2021 15:41:05 -0700 Subject: [PATCH 035/291] Cleaning up group call state --- src/client.ts | 27 +++++- src/webrtc/groupCall.ts | 123 ++++++++++++++++++---------- src/webrtc/groupCallEventHandler.ts | 77 ++++++++--------- 3 files changed, 148 insertions(+), 79 deletions(-) diff --git a/src/client.ts b/src/client.ts index a7ec0e4532d..fcaf901f20c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1337,7 +1337,32 @@ export class MatrixClient extends EventEmitter { * @returns {GroupCall} The group call or null if it doesn't already exist. */ public getGroupCallForRoom(roomId: string): GroupCall | null { - return this.groupCallEventHandler.groupCalls.get(roomId) || null; + const groupCall = this.groupCallEventHandler.groupCalls.get(roomId); + + if (groupCall) { + return groupCall; + } + + const room = this.getRoom(roomId); + + if (!room) { + return null; + } + + const stateEvents = room.currentState.getStateEvents(CONF_ROOM); + + if (stateEvents.length === 0) { + return null; + } + + const stateEvent = stateEvents[0]; + const content = stateEvent.getContent(); + + if (content.active) { + return this.groupCallEventHandler.createGroupCallFromRoomStateEvent(stateEvent); + } + + return null; } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 65c12caaa8d..af4e967025a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -10,8 +10,7 @@ import { ReEmitter } from "../ReEmitter"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; export enum GroupCallEvent { - Entered = "entered", - Left = "left", + GroupCallStateChanged = "group_call_state_changed", ActiveSpeakerChanged = "active_speaker_changed", CallsChanged = "calls_changed", UserMediaFeedsChanged = "user_media_feeds_changed", @@ -28,6 +27,15 @@ export interface IGroupCallDataChannelOptions { protocol: string; } +export enum GroupCallState { + LocalCallFeedUninitialized = "local_call_feed_uninitialized", + InitializingLocalCallFeed = "initializing_local_call_feed", + LocalCallFeedInitialized = "local_call_feed_initialized", + Entering = "entering", + Entered = "entered", + Ended = "ended", +} + interface IUserMediaFeedHandlers { onCallFeedVolumeChanged: (maxVolume: number) => void; onCallFeedMuteStateChanged: (audioMuted: boolean) => void; @@ -39,6 +47,10 @@ interface ICallHandlers { onCallHangup: (call: MatrixCall) => void; } +function getCallUserId(call: MatrixCall): string | null { + return call.getOpponentMember()?.userId || call.invitee || null; +} + export class GroupCall extends EventEmitter { // Config public activeSpeakerSampleCount = 8; @@ -46,7 +58,7 @@ export class GroupCall extends EventEmitter { public speakingThreshold = -80; public participantTimeout = 1000 * 15; - public entered = false; + public state = GroupCallState.LocalCallFeedUninitialized; public activeSpeaker: string; // userId public localCallFeed: CallFeed; public calls: MatrixCall[] = []; @@ -73,10 +85,12 @@ export class GroupCall extends EventEmitter { } public async initLocalCallFeed(): Promise { - if (this.localCallFeed) { - return this.localCallFeed; + if (this.state !== GroupCallState.LocalCallFeedUninitialized) { + throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } + this.state = GroupCallState.InitializingLocalCallFeed; + const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === CallType.Video); const userId = this.client.getUserId(); @@ -102,7 +116,12 @@ export class GroupCall extends EventEmitter { } public async enter() { - if (!this.localCallFeed) { + if (!(this.state === GroupCallState.LocalCallFeedUninitialized || + this.state === GroupCallState.LocalCallFeedInitialized)) { + throw new Error(`Cannot enter call in the "${this.state}" state`); + } + + if (this.state === GroupCallState.LocalCallFeedUninitialized) { await this.initLocalCallFeed(); } @@ -112,7 +131,7 @@ export class GroupCall extends EventEmitter { // Continue doing so every PARTICIPANT_TIMEOUT ms this.onPresenceLoop(); - this.entered = true; + this.state = GroupCallState.Entered; this.processInitialCalls(); @@ -127,11 +146,11 @@ export class GroupCall extends EventEmitter { this.client.on("RoomState.members", this.onRoomStateMembers); this.client.on("Call.incoming", this.onIncomingCall); - this.emit(GroupCallEvent.Entered); + this.emit(GroupCallEvent.GroupCallStateChanged, this.state); this.onActiveSpeakerLoop(); } - public leave() { + private dispose() { if (this.localCallFeed) { this.removeUserMediaFeed(this.localCallFeed); this.localCallFeed = null; @@ -139,7 +158,7 @@ export class GroupCall extends EventEmitter { this.client.getMediaHandler().stopAllStreams(); - if (!this.entered) { + if (this.state !== GroupCallState.Entered) { return; } @@ -160,11 +179,9 @@ export class GroupCall extends EventEmitter { ); while (this.calls.length > 0) { - const call = this.calls.pop(); - this.removeCall(call, CallErrorCode.UserHangup); + this.removeCall(this.calls[this.calls.length - 1], CallErrorCode.UserHangup); } - this.entered = false; this.activeSpeaker = null; clearTimeout(this.presenceLoopTimeout); clearTimeout(this.activeSpeakerLoopTimeout); @@ -174,23 +191,31 @@ export class GroupCall extends EventEmitter { this.onRoomStateMembers, ); this.client.removeListener("Call.incoming", this.onIncomingCall); + } - this.emit(GroupCallEvent.Left); + public leave() { + this.dispose(); + this.state = GroupCallState.LocalCallFeedUninitialized; + this.emit(GroupCallEvent.GroupCallStateChanged, this.state); } - public async endCall() { - this.leave(); + public async endCall(emitStateEvent = true) { + this.dispose(); + this.state = GroupCallState.Ended; this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); - this.client.emit("GroupCall.ended", this); + if (emitStateEvent) { + await this.client.sendStateEvent( + this.room.roomId, + CONF_ROOM, + { active: false }, + "", + ); + } - await this.client.sendStateEvent( - this.room.roomId, - CONF_ROOM, - { active: false }, - "", - ); + this.client.emit("GroupCall.ended", this); + this.emit(GroupCallEvent.GroupCallStateChanged, this.state); } /** @@ -269,7 +294,12 @@ export class GroupCall extends EventEmitter { for (let i = this.calls.length - 1; i >= 0; i--) { const call = this.calls[i]; - const opponentUserId = call.getOpponentMember().userId; + const opponentUserId = getCallUserId(call); + + if (!opponentUserId) { + continue; + } + const memberStateEvent = this.room.currentState.getStateEvents( "m.room.member", opponentUserId, @@ -435,7 +465,7 @@ export class GroupCall extends EventEmitter { */ public getCallByUserId(userId: string): MatrixCall { - return this.calls.find((call) => call.getOpponentMember().userId === userId); + return this.calls.find((call) => getCallUserId(call) === userId); } private addCall(call: MatrixCall, sessionId: string) { @@ -474,9 +504,13 @@ export class GroupCall extends EventEmitter { } private initCall(call: MatrixCall, sessionId: string) { - const opponentMemberId = call.getOpponentMember().userId; + const opponentMemberId = getCallUserId(call); + + if (!opponentMemberId) { + throw new Error("Cannot init call without user id"); + } - const onCallFeedsChanged = (feeds: CallFeed[]) => this.onCallFeedsChanged(call, feeds); + const onCallFeedsChanged = () => this.onCallFeedsChanged(call); const onCallStateChanged = (state: CallState, oldState: CallState) => this.onCallStateChanged(call, state, oldState); const onCallHangup = this.onCallHangup; @@ -499,7 +533,11 @@ export class GroupCall extends EventEmitter { } private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { - const opponentMemberId = call.getOpponentMember().userId; + const opponentMemberId = getCallUserId(call); + + if (!opponentMemberId) { + throw new Error("Cannot dispose call without user id"); + } const { onCallFeedsChanged, @@ -527,23 +565,26 @@ export class GroupCall extends EventEmitter { this.sessionIds.delete(opponentMemberId); } - private onCallFeedsChanged = (call: MatrixCall, feeds: CallFeed[]) => { - const opponentMemberId = call.getOpponentMember().userId; - const currentUserMediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + private onCallFeedsChanged = (call: MatrixCall) => { + const opponentMemberId = getCallUserId(call); + + if (!opponentMemberId) { + throw new Error("Cannot change call feeds without user id"); + } - let newUserMediaFeed: CallFeed; + const currentUserMediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + const remoteUsermediaFeed = call.remoteUsermediaFeed; + const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; - for (const feed of feeds) { - if (feed.purpose === SDPStreamMetadataPurpose.Usermedia && feed !== currentUserMediaFeed) { - newUserMediaFeed = feed; - } + if (!remoteFeedChanged) { + return; } - if (!currentUserMediaFeed && newUserMediaFeed) { - this.addUserMediaFeed(newUserMediaFeed); - } else if (currentUserMediaFeed && newUserMediaFeed) { - this.replaceUserMediaFeed(currentUserMediaFeed, newUserMediaFeed); - } else if (currentUserMediaFeed && !newUserMediaFeed) { + if (!currentUserMediaFeed && remoteUsermediaFeed) { + this.addUserMediaFeed(remoteUsermediaFeed); + } else if (currentUserMediaFeed && remoteUsermediaFeed) { + this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); + } else if (currentUserMediaFeed && !remoteUsermediaFeed) { this.removeUserMediaFeed(currentUserMediaFeed); } }; diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 4226072138e..019b141ebf5 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -34,53 +34,56 @@ export class GroupCallEventHandler { this.client.removeListener("RoomState.events", this.onRoomStateChanged); } + public createGroupCallFromRoomStateEvent(event: MatrixEvent) { + const roomId = event.getRoomId(); + const content = event.getContent(); + + let callType: CallType; + + if (content.callType === "voice") { + callType = CallType.Voice; + } else { + callType = CallType.Video; + } + + const room = this.client.getRoom(event.getRoomId()); + + if (!room) { + logger.error(`Couldn't find room ${roomId} for GroupCall`); + return; + } + + let dataChannelOptions: IGroupCallDataChannelOptions | undefined; + + if (content?.dataChannelsEnabled && content?.dataChannelOptions) { + // Pull out just the dataChannelOptions we want to support. + const { ordered, maxPacketLifeTime, maxRetransmits, protocol } = content.dataChannelOptions; + dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol }; + } + + return new GroupCall( + this.client, + room, + callType, + content?.dataChannelsEnabled, + dataChannelOptions, + ); + } + private onRoomStateChanged = (_event: MatrixEvent, state: RoomState): void => { const groupCall = this.groupCalls.get(state.roomId); const confEvents = state.getStateEvents(CONF_ROOM); - let content; - if (confEvents.length > 0) { - content = confEvents[0].getContent(); - } + const confEvent = confEvents.length > 0 ? confEvents[0] : null; + const content = confEvent ? confEvent.getContent() : null; if (groupCall && content?.type !== groupCall.type) { // TODO: Handle the callType changing when the room state changes logger.warn(`The group call type changed for room: ${ state.roomId}. Changing the group call type is currently unsupported.`); } if (groupCall && !content?.active) { - groupCall.leave(); - this.groupCalls.delete(state.roomId); - this.client.emit("GroupCall.ended", groupCall); + groupCall.endCall(false); } else if (!groupCall && content?.active) { - let callType: CallType; - - if (content.callType === "voice") { - callType = CallType.Voice; - } else { - callType = CallType.Video; - } - - const room = this.client.getRoom(state.roomId); - - if (!room) { - logger.error(`Couldn't find room ${state.roomId} for GroupCall`); - return; - } - - let dataChannelOptions: IGroupCallDataChannelOptions | undefined; - - if (content?.dataChannelsEnabled && content?.dataChannelOptions) { - // Pull out just the dataChannelOptions we want to support. - const { ordered, maxPacketLifeTime, maxRetransmits, protocol } = content.dataChannelOptions; - dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol }; - } - - const groupCall = new GroupCall( - this.client, - room, - callType, - content?.dataChannelsEnabled, - dataChannelOptions, - ); + const groupCall = this.createGroupCallFromRoomStateEvent(confEvent); this.groupCalls.set(state.roomId, groupCall); this.client.emit("GroupCall.incoming", groupCall); } From fc07530434d035ba1248a609de6f4b7170fa8c81 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 11:32:41 +0200 Subject: [PATCH 036/291] Add useToDevice to CallOpts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 4 ++-- src/webrtc/call.ts | 9 +++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index fcaf901f20c..3541326fad9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1285,8 +1285,8 @@ export class MatrixClient extends EventEmitter { * @param {string} invitee The user to call in the given room. * @return {MatrixCall} the call or null if the browser doesn't support calling. */ - public createCall(roomId: string, invitee?: string): MatrixCall { - return createNewMatrixCall(this, roomId, { invitee }); + public createCall(roomId: string, invitee?: string, useToDevice?: boolean): MatrixCall { + return createNewMatrixCall(this, roomId, { invitee, useToDevice }); } /** diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4b70d21021d..64cdb4f6291 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -71,6 +71,7 @@ interface CallOpts { client?: any; // Fix when client is TSified forceTURN?: boolean; turnServers?: Array; + useToDevice?: boolean; } interface TurnServer { @@ -312,6 +313,8 @@ export class MatrixCall extends EventEmitter { private callLengthInterval: number; private callLength = 0; + private useToDevice: boolean; + constructor(opts: CallOpts) { super(); this.roomId = opts.roomId; @@ -319,6 +322,7 @@ export class MatrixCall extends EventEmitter { this.client = opts.client; this.forceTURN = opts.forceTURN; this.ourPartyId = this.client.deviceId; + this.useToDevice = opts.useToDevice; // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { @@ -2126,13 +2130,14 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO const optionsForceTURN = options ? options.forceTURN : false; - const opts = { + const opts: CallOpts = { client: client, roomId: roomId, - invitee: options && options.invitee, + invitee: options?.invitee, turnServers: client.getTurnServers(), // call level options forceTURN: client.forceTURN || optionsForceTURN, + useToDevice: options?.useToDevice, }; const call = new MatrixCall(opts); From 3280394bf93622c096e3e260296f7f089b97846b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:15:53 +0200 Subject: [PATCH 037/291] Figure out opponentMember from the userId rather than the sender MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This is because to-device messages don't have a sender Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 64cdb4f6291..fc44fd98b1b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2041,7 +2041,7 @@ export class MatrixCall extends EventEmitter { this.opponentPartyId = msg.party_id || null; } this.opponentCaps = msg.capabilities || {} as CallCapabilities; - this.opponentMember = ev.sender; + this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); } private async addBufferedIceCandidates(): Promise { From 71ca424712b751418b3e2941cfbdb373608a7076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:17:07 +0200 Subject: [PATCH 038/291] Allow for sending toDevice messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index fc44fd98b1b..3f3dee3eb24 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1750,12 +1750,23 @@ export class MatrixCall extends EventEmitter { * @param {Object} content * @return {Promise} */ - private sendVoipEvent(eventType: string, content: object): Promise { - return this.client.sendEvent(this.roomId, eventType, Object.assign({}, content, { + private sendVoipEvent(eventType: string, content: object): Promise { + const realContent = Object.assign({}, content, { version: VOIP_PROTO_VERSION, call_id: this.callId, party_id: this.ourPartyId, - })); + call_room_id: this.roomId, + }); + + if (this.useToDevice) { + return this.client.sendToDevice(eventType, { + [this.invitee || this.getOpponentMember().userId]: { + "*": realContent, + }, + }); + } else { + return this.client.sendEvent(this.roomId, eventType, realContent); + } } private queueCandidate(content: RTCIceCandidate): void { From 027bc6bfc95342df6b4b7dcc24401ca383b60843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:19:12 +0200 Subject: [PATCH 039/291] Handle incoming to-device call signalling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/callEventHandler.ts | 52 ++++++++++++++++++++++++++++------ 1 file changed, 43 insertions(+), 9 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 7af3242070a..953a943e528 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -20,6 +20,7 @@ import { createNewMatrixCall, MatrixCall, CallErrorCode, CallState, CallDirectio import { EventType } from '../@types/event'; import { MatrixClient } from '../client'; import { MCallAnswer, MCallHangupReject } from "./callEventTypes"; +import { SyncState } from "../sync.api"; // Don't ring unless we'd be ringing for at least 3 seconds: the user needs some // time to press the 'accept' button @@ -31,6 +32,8 @@ export class CallEventHandler { callEventBuffer: MatrixEvent[]; candidateEventsByCall: Map>; + private toDeviceCallEventBuffer: MatrixEvent[] = []; + constructor(client: MatrixClient) { this.client = client; this.calls = new Map(); @@ -47,17 +50,24 @@ export class CallEventHandler { } public start() { - this.client.on("sync", this.evaluateEventBuffer); + this.client.on("sync", this.onSync); this.client.on("Room.timeline", this.onRoomTimeline); + this.client.on("toDeviceEvent", this.onToDeviceEvent); } public stop() { - this.client.removeListener("sync", this.evaluateEventBuffer); + this.client.removeListener("sync", this.onSync); this.client.removeListener("Room.timeline", this.onRoomTimeline); + this.client.removeListener("toDeviceEvent", this.onToDeviceEvent); } - private evaluateEventBuffer = async () => { - if (this.client.getSyncState() === "SYNCING") { + private onSync = (): void => { + this.evaluateEventBuffer(); + this.evaluateToDeviceEventBuffer(); + }; + + private async evaluateEventBuffer() { + if (this.client.getSyncState() === SyncState.Syncing) { await Promise.all(this.callEventBuffer.map(event => { this.client.decryptEventIfNeeded(event); })); @@ -88,7 +98,7 @@ export class CallEventHandler { } this.callEventBuffer = []; } - }; + } private onRoomTimeline = (event: MatrixEvent) => { this.client.decryptEventIfNeeded(event); @@ -120,6 +130,25 @@ export class CallEventHandler { } }; + private onToDeviceEvent = (event: MatrixEvent): void => { + if (!this.eventIsACall(event)) return; + + this.toDeviceCallEventBuffer.push(event); + }; + + private async evaluateToDeviceEventBuffer(): Promise { + if (this.client.getSyncState() !== SyncState.Syncing) return; + + for (const event of this.toDeviceCallEventBuffer) { + try { + await this.handleCallEvent(event, true); + } catch (e) { + logger.error("Caught exception handling call event", e); + } + } + this.toDeviceCallEventBuffer = []; + } + private eventIsACall(event: MatrixEvent): boolean { const type = event.getType(); /** @@ -129,13 +158,16 @@ export class CallEventHandler { return type.startsWith("m.call.") || type.startsWith("org.matrix.call."); } - private async handleCallEvent(event: MatrixEvent) { + private async handleCallEvent(event: MatrixEvent, isToDevice?: boolean) { const content = event.getContent(); + const callRoomId = event.getRoomId() || content.call_room_id; const type = event.getType() as EventType; const weSentTheEvent = event.getSender() === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; //console.info("RECV %s content=%s", type, JSON.stringify(content)); + if (!callRoomId) return; + if (type === EventType.CallInvite) { // ignore invites you send if (weSentTheEvent) return; @@ -159,8 +191,8 @@ export class CallEventHandler { logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms"); call = createNewMatrixCall( this.client, - event.getRoomId(), - { forceTURN: this.client.forceTURN }, + callRoomId, + { forceTURN: this.client.forceTURN, useToDevice: isToDevice }, ); if (!call) { logger.log( @@ -249,7 +281,9 @@ export class CallEventHandler { // if not live, store the fact that the call has ended because // we're probably getting events backwards so // the hangup will come before the invite - call = createNewMatrixCall(this.client, event.getRoomId()); + call = createNewMatrixCall( + this.client, callRoomId, { useToDevice: isToDevice }, + ); if (call) { call.callId = content.call_id; call.initWithHangup(event); From b535969845eb345f2d5293d99adb0234fe9fa5cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:19:30 +0200 Subject: [PATCH 040/291] Use to-device messages in group calls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index af4e967025a..e222c65aa37 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -442,7 +442,7 @@ export class GroupCall extends EventEmitter { return; } - const newCall = this.client.createCall(this.room.roomId, member.userId); + const newCall = this.client.createCall(this.room.roomId, member.userId, true); // TODO: Move to call.placeCall() const callPromise = this.type === CallType.Video ? newCall.placeVideoCall() : newCall.placeVoiceCall(); From 678023717bab3522ad9156d8a9847e6e80ccb5f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:38:32 +0200 Subject: [PATCH 041/291] Add a setState() method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e222c65aa37..f6d21423432 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -84,12 +84,18 @@ export class GroupCall extends EventEmitter { this.reEmitter = new ReEmitter(this); } + private setState(newState: GroupCallState): void { + const oldState = this.state; + this.state = newState; + this.emit(GroupCallEvent.GroupCallStateChanged, newState, oldState); + } + public async initLocalCallFeed(): Promise { if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } - this.state = GroupCallState.InitializingLocalCallFeed; + this.setState(GroupCallState.InitializingLocalCallFeed); const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === CallType.Video); @@ -131,7 +137,7 @@ export class GroupCall extends EventEmitter { // Continue doing so every PARTICIPANT_TIMEOUT ms this.onPresenceLoop(); - this.state = GroupCallState.Entered; + this.setState(GroupCallState.Entered); this.processInitialCalls(); @@ -146,7 +152,6 @@ export class GroupCall extends EventEmitter { this.client.on("RoomState.members", this.onRoomStateMembers); this.client.on("Call.incoming", this.onIncomingCall); - this.emit(GroupCallEvent.GroupCallStateChanged, this.state); this.onActiveSpeakerLoop(); } @@ -195,13 +200,11 @@ export class GroupCall extends EventEmitter { public leave() { this.dispose(); - this.state = GroupCallState.LocalCallFeedUninitialized; - this.emit(GroupCallEvent.GroupCallStateChanged, this.state); + this.setState(GroupCallState.LocalCallFeedUninitialized); } public async endCall(emitStateEvent = true) { this.dispose(); - this.state = GroupCallState.Ended; this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); @@ -215,7 +218,7 @@ export class GroupCall extends EventEmitter { } this.client.emit("GroupCall.ended", this); - this.emit(GroupCallEvent.GroupCallStateChanged, this.state); + this.setState(GroupCallState.Ended); } /** From 205c80ea2864f92b1d2e85b4cfe0d9a7c78c10e6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:55:25 +0200 Subject: [PATCH 042/291] Add groupCallId MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 8 +++++++- src/webrtc/call.ts | 2 +- src/webrtc/groupCall.ts | 4 +++- src/webrtc/groupCallEventHandler.ts | 5 ++++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index 3541326fad9..d7d7794d2a9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1323,7 +1323,13 @@ export class MatrixClient extends EventEmitter { await this.sendStateEvent( room.roomId, CONF_ROOM, - { active: true, callType: type, dataChannelsEnabled, dataChannelOptions }, + { + active: true, + callType: type, + conf_id: groupCall.groupCallId, + dataChannelsEnabled, + dataChannelOptions, + }, "", ); } diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3f3dee3eb24..38707141cce 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -238,7 +238,7 @@ export class CallError extends Error { } } -function genCallID(): string { +export function genCallID(): string { return Date.now().toString() + randomString(16); } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index f6d21423432..d3b73af701a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -2,7 +2,7 @@ import EventEmitter from "events"; import { CallFeed, CallFeedEvent } from "./callFeed"; import { MatrixClient } from "../client"; import { randomString } from "../randomstring"; -import { CallErrorCode, CallEvent, CallState, CallType, MatrixCall, setTracksEnabled } from "./call"; +import { CallErrorCode, CallEvent, CallState, CallType, genCallID, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; @@ -63,6 +63,7 @@ export class GroupCall extends EventEmitter { public localCallFeed: CallFeed; public calls: MatrixCall[] = []; public userMediaFeeds: CallFeed[] = []; + public groupCallId: string; private userMediaFeedHandlers: Map = new Map(); private callHandlers: Map = new Map(); @@ -82,6 +83,7 @@ export class GroupCall extends EventEmitter { ) { super(); this.reEmitter = new ReEmitter(this); + this.groupCallId = genCallID(); } private setState(newState: GroupCallState): void { diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 019b141ebf5..b3b64df70ab 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -61,13 +61,16 @@ export class GroupCallEventHandler { dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol }; } - return new GroupCall( + const groupCall = new GroupCall( this.client, room, callType, content?.dataChannelsEnabled, dataChannelOptions, ); + groupCall.groupCallId = content["conf_id"]; + + return groupCall; } private onRoomStateChanged = (_event: MatrixEvent, state: RoomState): void => { From e9e65cf484cd898e089e4e6129dec81153fa242c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 16:58:11 +0200 Subject: [PATCH 043/291] Change type key MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client.ts b/src/client.ts index d7d7794d2a9..373942571e1 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1325,7 +1325,7 @@ export class MatrixClient extends EventEmitter { CONF_ROOM, { active: true, - callType: type, + ["m.type"]: type, conf_id: groupCall.groupCallId, dataChannelsEnabled, dataChannelOptions, From a2f23900c9fa756b015059a6d60ee23ff421d73c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sat, 25 Sep 2021 17:20:39 +0200 Subject: [PATCH 044/291] Use groupCallId isntead of roomId in to-device messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 4 ++-- src/webrtc/call.ts | 6 +++++- src/webrtc/callEventHandler.ts | 8 ++++++-- src/webrtc/groupCall.ts | 2 +- src/webrtc/groupCallEventHandler.ts | 4 ++++ 5 files changed, 18 insertions(+), 6 deletions(-) diff --git a/src/client.ts b/src/client.ts index 373942571e1..682b71807d6 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1285,8 +1285,8 @@ export class MatrixClient extends EventEmitter { * @param {string} invitee The user to call in the given room. * @return {MatrixCall} the call or null if the browser doesn't support calling. */ - public createCall(roomId: string, invitee?: string, useToDevice?: boolean): MatrixCall { - return createNewMatrixCall(this, roomId, { invitee, useToDevice }); + public createCall(roomId: string, invitee?: string, useToDevice?: boolean, groupCallId?: string): MatrixCall { + return createNewMatrixCall(this, roomId, { invitee, useToDevice, groupCallId }); } /** diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 38707141cce..ae9aa1f8464 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -72,6 +72,7 @@ interface CallOpts { forceTURN?: boolean; turnServers?: Array; useToDevice?: boolean; + groupCallId?: string; } interface TurnServer { @@ -314,6 +315,7 @@ export class MatrixCall extends EventEmitter { private callLength = 0; private useToDevice: boolean; + private groupCallId: string; constructor(opts: CallOpts) { super(); @@ -323,6 +325,7 @@ export class MatrixCall extends EventEmitter { this.forceTURN = opts.forceTURN; this.ourPartyId = this.client.deviceId; this.useToDevice = opts.useToDevice; + this.groupCallId = opts.groupCallId; // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { @@ -1755,7 +1758,7 @@ export class MatrixCall extends EventEmitter { version: VOIP_PROTO_VERSION, call_id: this.callId, party_id: this.ourPartyId, - call_room_id: this.roomId, + conf_id: this.groupCallId, }); if (this.useToDevice) { @@ -2149,6 +2152,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO // call level options forceTURN: client.forceTURN || optionsForceTURN, useToDevice: options?.useToDevice, + groupCallId: options?.groupCallId, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 953a943e528..377b5fe646d 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -160,7 +160,11 @@ export class CallEventHandler { private async handleCallEvent(event: MatrixEvent, isToDevice?: boolean) { const content = event.getContent(); - const callRoomId = event.getRoomId() || content.call_room_id; + const callRoomId = ( + event.getRoomId() || + this.client.groupCallEventHandler.getGroupCallById(content.conf_id)?.room?.roomId + ); + const groupCallId = content.conf_id; const type = event.getType() as EventType; const weSentTheEvent = event.getSender() === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; @@ -192,7 +196,7 @@ export class CallEventHandler { call = createNewMatrixCall( this.client, callRoomId, - { forceTURN: this.client.forceTURN, useToDevice: isToDevice }, + { forceTURN: this.client.forceTURN, useToDevice: isToDevice, groupCallId }, ); if (!call) { logger.log( diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d3b73af701a..b4f2b5eda3a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -447,7 +447,7 @@ export class GroupCall extends EventEmitter { return; } - const newCall = this.client.createCall(this.room.roomId, member.userId, true); + const newCall = this.client.createCall(this.room.roomId, member.userId, true, this.groupCallId); // TODO: Move to call.placeCall() const callPromise = this.type === CallType.Video ? newCall.placeVideoCall() : newCall.placeVoiceCall(); diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index b3b64df70ab..00f99624cfb 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -34,6 +34,10 @@ export class GroupCallEventHandler { this.client.removeListener("RoomState.events", this.onRoomStateChanged); } + public getGroupCallById(groupCallId: string): GroupCall { + return [...this.groupCalls.values()].find((groupCall) => groupCall.groupCallId === groupCallId); + } + public createGroupCallFromRoomStateEvent(event: MatrixEvent) { const roomId = event.getRoomId(); const content = event.getContent(); From 715c4577d0d26ca37e08a96adc7c7bb8745618e2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 26 Sep 2021 11:35:41 +0200 Subject: [PATCH 045/291] Add a prop for not stopping local feeds on end MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ae9aa1f8464..059b2a73049 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -317,6 +317,11 @@ export class MatrixCall extends EventEmitter { private useToDevice: boolean; private groupCallId: string; + // Whether to stop local media when the call ends. We do not want to do this + // in group calls where the media is supplied by the group call. Doing this + // in a group call would result in breaking all the other calls + private stopLocalMediaOnEnd = true; + constructor(opts: CallOpts) { super(); this.roomId = opts.roomId; @@ -603,7 +608,9 @@ export class MatrixCall extends EventEmitter { private deleteAllFeeds(): void { for (const feed of this.feeds) { - feed.dispose(); + if (!feed.isLocal() || this.stopLocalMediaOnEnd) { + feed.dispose(); + } } this.feeds = []; @@ -1899,14 +1906,22 @@ export class MatrixCall extends EventEmitter { } private stopAllMedia(): void { - logger.debug(`stopAllMedia (stream=${this.localUsermediaStream})`); + logger.debug(this.stopLocalMediaOnEnd ? "Stopping all media" : "Stopping all media except local feeds" ); for (const feed of this.feeds) { - if (feed.isLocal() && feed.purpose === SDPStreamMetadataPurpose.Usermedia) { + if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Usermedia && + this.stopLocalMediaOnEnd + ) { this.client.getMediaHandler().stopUserMediaStream(feed.stream); - } else if (feed.isLocal() && feed.purpose === SDPStreamMetadataPurpose.Screenshare) { + } else if ( + feed.isLocal() && + feed.purpose === SDPStreamMetadataPurpose.Screenshare && + this.stopLocalMediaOnEnd + ) { this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else { + } else if (!feed.isLocal() || this.stopLocalMediaOnEnd) { for (const track of feed.stream.getTracks()) { track.stop(); } From 2515ba31a083ccfc24137c8434663eda7113b7b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 26 Sep 2021 12:04:00 +0200 Subject: [PATCH 046/291] Use createNewMatrixCall() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/client.ts | 5 ++--- src/webrtc/groupCall.ts | 7 ++++++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index 682b71807d6..27875ff58cc 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1282,11 +1282,10 @@ export class MatrixClient extends EventEmitter { * The place*Call methods on the returned call can be used to actually place a call * * @param {string} roomId The room the call is to be placed in. - * @param {string} invitee The user to call in the given room. * @return {MatrixCall} the call or null if the browser doesn't support calling. */ - public createCall(roomId: string, invitee?: string, useToDevice?: boolean, groupCallId?: string): MatrixCall { - return createNewMatrixCall(this, roomId, { invitee, useToDevice, groupCallId }); + public createCall(roomId: string): MatrixCall { + return createNewMatrixCall(this, roomId); } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b4f2b5eda3a..6466111596d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -8,6 +8,7 @@ import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { createNewMatrixCall } from "./call"; export enum GroupCallEvent { GroupCallStateChanged = "group_call_state_changed", @@ -447,7 +448,11 @@ export class GroupCall extends EventEmitter { return; } - const newCall = this.client.createCall(this.room.roomId, member.userId, true, this.groupCallId); + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { invitee: member.userId, useToDevice: true, groupCallId: this.groupCallId }, + ); // TODO: Move to call.placeCall() const callPromise = this.type === CallType.Video ? newCall.placeVideoCall() : newCall.placeVoiceCall(); From d8ef7f9f6379a75d9103bf297cce319f3715f2d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 26 Sep 2021 12:08:52 +0200 Subject: [PATCH 047/291] pushLocalFeed() -> pushNewLocalFeed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 059b2a73049..47e65e734bd 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -567,7 +567,7 @@ export class MatrixCall extends EventEmitter { logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); } - private pushLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { + private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { const userId = this.client.getUserId(); // We try to replace an existing feed if there already is one with the same purpose @@ -864,7 +864,7 @@ export class MatrixCall extends EventEmitter { if (this.hasLocalUserMediaAudioTrack) return; if (this.hasLocalUserMediaVideoTrack) return; - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); } else if (upgradeAudio) { if (this.hasLocalUserMediaAudioTrack) return; @@ -930,7 +930,7 @@ export class MatrixCall extends EventEmitter { try { const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); if (!stream) return false; - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { this.emit(CallEvent.Error, @@ -972,7 +972,7 @@ export class MatrixCall extends EventEmitter { }); sender.replaceTrack(track); - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); return true; } catch (err) { @@ -1138,7 +1138,7 @@ export class MatrixCall extends EventEmitter { return; } - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); this.setState(CallState.CreateOffer); logger.debug("gotUserMediaForInvite"); @@ -1197,7 +1197,7 @@ export class MatrixCall extends EventEmitter { return; } - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); + this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); this.setState(CallState.CreateAnswer); let myAnswer; From df208e4de8fc8d122c273a3f1f3c4fc3da28e531 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 26 Sep 2021 12:51:23 +0200 Subject: [PATCH 048/291] Avoid having duplicate call feeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 150 +++++++++++++++++++++++++++++++++++++++- src/webrtc/groupCall.ts | 13 ++-- 2 files changed, 154 insertions(+), 9 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 47e65e734bd..01c7d04e449 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -606,6 +606,43 @@ export class MatrixCall extends EventEmitter { logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`); } + /** + * Pushes supplied feed to the call + * @param {CallFeed} callFeed to push + * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection + */ + public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + this.feeds.push(callFeed); + this.emit(CallEvent.FeedsChanged, this.feeds); + + if (addToPeerConnection) { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? + this.usermediaSenders : this.screensharingSenders; + // Empty the array + senderArray.splice(0, senderArray.length); + + this.emit(CallEvent.FeedsChanged, this.feeds); + for (const track of callFeed.stream.getTracks()) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${callFeed.stream}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); + } + } + + logger.info( + `Pushed local stream `+ + `(id="${callFeed.stream.id}", `+ + `active="${callFeed.stream.active}", `+ + `purpose="${callFeed.purpose}")`, + ); + } + private deleteAllFeeds(): void { for (const feed of this.feeds) { if (!feed.isLocal() || this.stopLocalMediaOnEnd) { @@ -781,6 +818,28 @@ export class MatrixCall extends EventEmitter { } } + public answerWithCallFeed(callFeed: CallFeed): void { + this.stopLocalMediaOnEnd = false; + if (this.inviteOrAnswerSent) return; + + logger.debug(`Answering call ${this.callId}`); + + if (!this.localUsermediaStream && !this.waitForLocalAVStream) { + this.setState(CallState.WaitLocalMedia); + this.waitForLocalAVStream = true; + + try { + this.waitForLocalAVStream = false; + this.gotCallFeedForAnswer(callFeed); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } else if (this.waitForLocalAVStream) { + this.setState(CallState.WaitLocalMedia); + } + } + /** * Replace this call with a new call, e.g. for glare resolution. Used by * MatrixClient. @@ -792,7 +851,11 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { logger.debug("Handing local stream to new call"); - newCall.gotUserMediaForAnswer(this.localUsermediaStream); + if (this.stopLocalMediaOnEnd) { + newCall.gotUserMediaForAnswer(this.localUsermediaStream); + } else { + newCall.gotCallFeedForAnswer(this.localUsermediaFeed); + } } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -1145,6 +1208,23 @@ export class MatrixCall extends EventEmitter { // Now we wait for the negotiationneeded event }; + private gotCallFeedForInvite(callFeed: CallFeed): void { + if (this.successor) { + this.successor.gotCallFeedForAnswer(callFeed); + return; + } + if (this.callHasEnded()) { + this.stopAllMedia(); + return; + } + + this.pushLocalFeed(callFeed); + this.setState(CallState.CreateOffer); + + logger.debug("gotUserMediaForInvite"); + // Now we wait for the negotiationneeded event + } + private async sendAnswer(): Promise { const answerContent = { answer: { @@ -1227,6 +1307,41 @@ export class MatrixCall extends EventEmitter { } }; + private async gotCallFeedForAnswer(callFeed: CallFeed): Promise { + if (this.callHasEnded()) return; + + this.waitForLocalAVStream = false; + + this.pushLocalFeed(callFeed); + this.setState(CallState.CreateAnswer); + + let myAnswer; + try { + this.getRidOfRTXCodecs(); + myAnswer = await this.peerConn.createAnswer(); + } catch (err) { + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); + return; + } + + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + + // Allow a short time for initial candidates to be gathered + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + this.sendAnswer(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } + } + /** * Internal * @param {Object} event @@ -2022,6 +2137,39 @@ export class MatrixCall extends EventEmitter { } } + /** + * Place a call to this room with call feed. + * @param {CallFeed} callFeed to use + * @throws if you have not specified a listener for 'error' events. + * @throws if have passed audio=false. + */ + public async placeCallWithCallFeed(callFeed: CallFeed): Promise { + this.stopLocalMediaOnEnd = false; + this.checkForErrorListener(); + // XXX Find a better way to do this + this.client.callEventHandler.calls.set(this.callId, this); + this.setState(CallState.WaitLocalMedia); + this.direction = CallDirection.Outbound; + + // make sure we have valid turn creds. Unless something's gone wrong, it should + // poll and keep the credentials valid so this should be instant. + const haveTurnCreds = await this.client.checkTurnServers(); + if (!haveTurnCreds) { + logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + } + + // create the peer connection now so it can be gathering candidates while we get user + // media (assuming a candidate pool size is configured) + this.peerConn = this.createPeerConnection(); + + try { + this.gotCallFeedForInvite(callFeed); + } catch (e) { + this.getUserMediaFailed(e); + return; + } + } + private createPeerConnection(): RTCPeerConnection { const pc = new window.RTCPeerConnection({ iceTransportPolicy: this.forceTURN ? 'relay' : undefined, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 6466111596d..0019c941ff1 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -386,7 +386,7 @@ export class GroupCall extends EventEmitter { this.addCall(newCall, sessionId); } - newCall.answer(); + newCall.answerWithCallFeed(this.localCallFeed); }; private onRoomStateMembers = (_event, _state, member: RoomMember) => { @@ -454,14 +454,11 @@ export class GroupCall extends EventEmitter { { invitee: member.userId, useToDevice: true, groupCallId: this.groupCallId }, ); - // TODO: Move to call.placeCall() - const callPromise = this.type === CallType.Video ? newCall.placeVideoCall() : newCall.placeVoiceCall(); + newCall.placeCallWithCallFeed(this.localCallFeed); - callPromise.then(() => { - if (this.dataChannelsEnabled) { - newCall.createDataChannel("datachannel", this.dataChannelOptions); - } - }); + if (this.dataChannelsEnabled) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } if (existingCall) { this.replaceCall(existingCall, newCall, sessionId); From cf1ba1223210926cb23e44c9d1f14784931db8d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 26 Sep 2021 13:02:57 +0200 Subject: [PATCH 049/291] Use arrays of CallFeeds MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 27 ++++++++++++++++----------- src/webrtc/groupCall.ts | 4 ++-- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 01c7d04e449..34afa80acfc 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -818,7 +818,7 @@ export class MatrixCall extends EventEmitter { } } - public answerWithCallFeed(callFeed: CallFeed): void { + public answerWithCallFeeds(callFeeds: CallFeed[]): void { this.stopLocalMediaOnEnd = false; if (this.inviteOrAnswerSent) return; @@ -830,7 +830,7 @@ export class MatrixCall extends EventEmitter { try { this.waitForLocalAVStream = false; - this.gotCallFeedForAnswer(callFeed); + this.gotCallFeedsForAnswer(callFeeds); } catch (e) { this.getUserMediaFailed(e); return; @@ -854,7 +854,7 @@ export class MatrixCall extends EventEmitter { if (this.stopLocalMediaOnEnd) { newCall.gotUserMediaForAnswer(this.localUsermediaStream); } else { - newCall.gotCallFeedForAnswer(this.localUsermediaFeed); + newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); } } this.successor = newCall; @@ -1208,9 +1208,9 @@ export class MatrixCall extends EventEmitter { // Now we wait for the negotiationneeded event }; - private gotCallFeedForInvite(callFeed: CallFeed): void { + private gotCallFeedsForInvite(callFeeds: CallFeed[]): void { if (this.successor) { - this.successor.gotCallFeedForAnswer(callFeed); + this.successor.gotCallFeedsForAnswer(callFeeds); return; } if (this.callHasEnded()) { @@ -1218,7 +1218,9 @@ export class MatrixCall extends EventEmitter { return; } - this.pushLocalFeed(callFeed); + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } this.setState(CallState.CreateOffer); logger.debug("gotUserMediaForInvite"); @@ -1307,12 +1309,15 @@ export class MatrixCall extends EventEmitter { } }; - private async gotCallFeedForAnswer(callFeed: CallFeed): Promise { + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { if (this.callHasEnded()) return; this.waitForLocalAVStream = false; - this.pushLocalFeed(callFeed); + for (const feed of callFeeds) { + this.pushLocalFeed(feed); + } + this.setState(CallState.CreateAnswer); let myAnswer; @@ -2139,11 +2144,11 @@ export class MatrixCall extends EventEmitter { /** * Place a call to this room with call feed. - * @param {CallFeed} callFeed to use + * @param {CallFeed[]} callFeeds to use * @throws if you have not specified a listener for 'error' events. * @throws if have passed audio=false. */ - public async placeCallWithCallFeed(callFeed: CallFeed): Promise { + public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise { this.stopLocalMediaOnEnd = false; this.checkForErrorListener(); // XXX Find a better way to do this @@ -2163,7 +2168,7 @@ export class MatrixCall extends EventEmitter { this.peerConn = this.createPeerConnection(); try { - this.gotCallFeedForInvite(callFeed); + this.gotCallFeedsForInvite(callFeeds); } catch (e) { this.getUserMediaFailed(e); return; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0019c941ff1..6a498bd0f52 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -386,7 +386,7 @@ export class GroupCall extends EventEmitter { this.addCall(newCall, sessionId); } - newCall.answerWithCallFeed(this.localCallFeed); + newCall.answerWithCallFeeds([this.localCallFeed]); }; private onRoomStateMembers = (_event, _state, member: RoomMember) => { @@ -454,7 +454,7 @@ export class GroupCall extends EventEmitter { { invitee: member.userId, useToDevice: true, groupCallId: this.groupCallId }, ); - newCall.placeCallWithCallFeed(this.localCallFeed); + newCall.placeCallWithCallFeeds([this.localCallFeed]); if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); From f85aa44f283f52b4d518f2a6ede069ec1e5fb0ad Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 27 Sep 2021 15:02:59 -0700 Subject: [PATCH 050/291] Remove duplicate FeedChanged event --- src/webrtc/call.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1ca1b7cb8b4..2789808b6b9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -613,7 +613,6 @@ export class MatrixCall extends EventEmitter { */ public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { this.feeds.push(callFeed); - this.emit(CallEvent.FeedsChanged, this.feeds); if (addToPeerConnection) { const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? @@ -621,7 +620,6 @@ export class MatrixCall extends EventEmitter { // Empty the array senderArray.splice(0, senderArray.length); - this.emit(CallEvent.FeedsChanged, this.feeds); for (const track of callFeed.stream.getTracks()) { logger.info( `Adding track (` + @@ -641,6 +639,8 @@ export class MatrixCall extends EventEmitter { `active="${callFeed.stream.active}", `+ `purpose="${callFeed.purpose}")`, ); + + this.emit(CallEvent.FeedsChanged, this.feeds); } private deleteAllFeeds(): void { From 7a142e9102380c3012224120c318a0d7745a646a Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 27 Sep 2021 17:06:09 -0700 Subject: [PATCH 051/291] Implement new group call state events --- src/client.ts | 62 +++---- src/webrtc/call.ts | 2 +- src/webrtc/groupCall.ts | 250 +++++++++++----------------- src/webrtc/groupCallEventHandler.ts | 68 +++++--- 4 files changed, 176 insertions(+), 206 deletions(-) diff --git a/src/client.ts b/src/client.ts index 27875ff58cc..027d105fb2d 100644 --- a/src/client.ts +++ b/src/client.ts @@ -23,7 +23,7 @@ import { EventEmitter } from "events"; import { ISyncStateData, SyncApi } from "./sync"; import { EventStatus, IContent, IDecryptOptions, IEvent, MatrixEvent } from "./models/event"; import { StubStore } from "./store/stub"; -import { createNewMatrixCall, MatrixCall, CallType } from "./webrtc/call"; +import { createNewMatrixCall, MatrixCall } from "./webrtc/call"; import { Filter, IFilterDefinition } from "./filter"; import { CallEventHandler } from './webrtc/callEventHandler'; import * as utils from './utils'; @@ -144,7 +144,13 @@ import { IHierarchyRoom, ISpaceSummaryEvent, ISpaceSummaryRoom } from "./@types/ import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; -import { CONF_ROOM, GroupCall, IGroupCallDataChannelOptions } from "./webrtc/groupCall"; +import { + CALL_EVENT, + GroupCall, + IGroupCallDataChannelOptions, + GroupCallIntent, + GroupCallType, +} from "./webrtc/groupCall"; import { MediaHandler } from "./webrtc/mediaHandler"; import { GroupCallEventHandler } from "./webrtc/groupCallEventHandler"; @@ -1297,7 +1303,8 @@ export class MatrixClient extends EventEmitter { */ public async createGroupCall( roomId: string, - type: CallType, + type: GroupCallType, + intent: GroupCallIntent, dataChannelsEnabled?: boolean, dataChannelOptions?: IGroupCallDataChannelOptions, ): Promise { @@ -1311,27 +1318,21 @@ export class MatrixClient extends EventEmitter { throw new Error(`Cannot find room ${roomId}`); } - const groupCall = new GroupCall(this, room, type, dataChannelsEnabled, dataChannelOptions); + const groupCall = new GroupCall(this, room, type, intent, dataChannelsEnabled, dataChannelOptions); this.groupCallEventHandler.groupCalls.set(roomId, groupCall); - const activeConf = room.currentState - .getStateEvents(CONF_ROOM, "") - ?.getContent()?.active; - - if (!activeConf) { - await this.sendStateEvent( - room.roomId, - CONF_ROOM, - { - active: true, - ["m.type"]: type, - conf_id: groupCall.groupCallId, - dataChannelsEnabled, - dataChannelOptions, - }, - "", - ); - } + await this.sendStateEvent( + room.roomId, + CALL_EVENT, + { + "m.intent": intent, + "m.type": type, + // TODO: Specify datachannels + dataChannelsEnabled, + dataChannelOptions, + }, + groupCall.groupCallId, + ); return groupCall; } @@ -1354,17 +1355,22 @@ export class MatrixClient extends EventEmitter { return null; } - const stateEvents = room.currentState.getStateEvents(CONF_ROOM); + const callEvents = room.currentState.getStateEvents(CALL_EVENT); - if (stateEvents.length === 0) { + if (callEvents.length === 0) { return null; } - const stateEvent = stateEvents[0]; - const content = stateEvent.getContent(); + const sortedCallEvents = callEvents.sort((a, b) => a.getTs() - b.getTs()); + + for (const callEvent of sortedCallEvents) { + const content = callEvent.getContent(); + + if (content["m.terminated"]) { + continue; + } - if (content.active) { - return this.groupCallEventHandler.createGroupCallFromRoomStateEvent(stateEvent); + return this.groupCallEventHandler.createGroupCallFromRoomStateEvent(callEvent); } return null; diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2789808b6b9..dfddbaca8e2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -315,7 +315,7 @@ export class MatrixCall extends EventEmitter { private callLength = 0; private useToDevice: boolean; - private groupCallId: string; + public groupCallId: string; // Whether to stop local media when the call ends. We do not want to do this // in group calls where the media is supplied by the group call. Doing this diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 6a498bd0f52..e2cc66d261e 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,14 +1,34 @@ import EventEmitter from "events"; import { CallFeed, CallFeedEvent } from "./callFeed"; import { MatrixClient } from "../client"; -import { randomString } from "../randomstring"; -import { CallErrorCode, CallEvent, CallState, CallType, genCallID, MatrixCall, setTracksEnabled } from "./call"; +import { CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { createNewMatrixCall } from "./call"; +import { ISendEventResponse } from "../@types/requests"; +import { MatrixEvent } from "../models/event"; +import { RoomState } from "../models/room-state"; + +export const CALL_EVENT = "org.matrix.msc3401.call"; +export const CALL_MEMBER_KEY = "org.matrix.msc3401.calls"; + +export enum GroupCallIntent { + Ring = "m.ring", + Prompt = "m.propmt", + Room = "m.room", +} + +export enum GroupCallType { + Video = "m.video", + Voice = "m.voice", +} + +export enum GroupCallTerminationReason { + CallEnded = "call_ended", +} export enum GroupCallEvent { GroupCallStateChanged = "group_call_state_changed", @@ -18,9 +38,6 @@ export enum GroupCallEvent { LocalMuteStateChanged = "local_mute_state_changed", } -export const CONF_ROOM = "me.robertlong.conf"; -const CONF_PARTICIPANT = "me.robertlong.conf.participant"; - export interface IGroupCallDataChannelOptions { ordered: boolean; maxPacketLifeTime: number; @@ -68,17 +85,15 @@ export class GroupCall extends EventEmitter { private userMediaFeedHandlers: Map = new Map(); private callHandlers: Map = new Map(); - private sessionIds: Map = new Map(); // userId -> sessionId private activeSpeakerSamples: Map = new Map(); - - private presenceLoopTimeout?: number; private activeSpeakerLoopTimeout?: number; private reEmitter: ReEmitter; constructor( private client: MatrixClient, public room: Room, - public type: CallType, + public type: GroupCallType, + public intent: GroupCallIntent, private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, ) { @@ -100,7 +115,7 @@ export class GroupCall extends EventEmitter { this.setState(GroupCallState.InitializingLocalCallFeed); - const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === CallType.Video); + const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); const userId = this.client.getUserId(); @@ -114,7 +129,6 @@ export class GroupCall extends EventEmitter { false, ); - this.sessionIds.set(userId, randomString(16)); this.activeSpeakerSamples.set(userId, Array(this.activeSpeakerSampleCount).fill( -Infinity, )); @@ -134,11 +148,9 @@ export class GroupCall extends EventEmitter { await this.initLocalCallFeed(); } - this.activeSpeaker = this.client.getUserId(); + this.sendEnteredMemberStateEvent(); - // Announce to the other room members that we have entered the room. - // Continue doing so every PARTICIPANT_TIMEOUT ms - this.onPresenceLoop(); + this.activeSpeaker = this.client.getUserId(); this.setState(GroupCallState.Entered); @@ -146,13 +158,15 @@ export class GroupCall extends EventEmitter { // Set up participants for the members currently in the room. // Other members will be picked up by the RoomState.members event. - const initialMembers = this.room.getMembers(); + const roomState = this.room.currentState; + const memberStateEvents = roomState.getStateEvents("m.room.member"); - for (const member of initialMembers) { - this.onMemberChanged(member); + for (const stateEvent of memberStateEvents) { + const member = this.room.getMember(stateEvent.getStateKey()); + this.onMemberStateChanged(stateEvent, roomState, member); } - this.client.on("RoomState.members", this.onRoomStateMembers); + this.client.on("RoomState.members", this.onMemberStateChanged); this.client.on("Call.incoming", this.onIncomingCall); this.onActiveSpeakerLoop(); @@ -170,33 +184,18 @@ export class GroupCall extends EventEmitter { return; } - const userId = this.client.getUserId(); - const currentMemberState = this.room.currentState.getStateEvents( - "m.room.member", - userId, - ); - - this.client.sendStateEvent( - this.room.roomId, - "m.room.member", - { - ...currentMemberState.getContent(), - [CONF_PARTICIPANT]: null, - }, - userId, - ); + this.sendLeftMemberStateEvent(); while (this.calls.length > 0) { this.removeCall(this.calls[this.calls.length - 1], CallErrorCode.UserHangup); } this.activeSpeaker = null; - clearTimeout(this.presenceLoopTimeout); clearTimeout(this.activeSpeakerLoopTimeout); this.client.removeListener( "RoomState.members", - this.onRoomStateMembers, + this.onMemberStateChanged, ); this.client.removeListener("Call.incoming", this.onIncomingCall); } @@ -206,17 +205,22 @@ export class GroupCall extends EventEmitter { this.setState(GroupCallState.LocalCallFeedUninitialized); } - public async endCall(emitStateEvent = true) { + public async terminate(emitStateEvent = true) { this.dispose(); this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); if (emitStateEvent) { + const existingStateEvent = this.room.currentState.getStateEvents(CALL_EVENT, this.groupCallId); + await this.client.sendStateEvent( this.room.roomId, - CONF_ROOM, - { active: false }, - "", + CALL_EVENT, + { + ...existingStateEvent.getContent(), + ["m.terminated"]: GroupCallTerminationReason.CallEnded, + }, + this.groupCallId, ); } @@ -270,66 +274,6 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); } - /** - * Call presence - */ - - private onPresenceLoop = () => { - const localUserId = this.client.getUserId(); - const currentMemberState = this.room.currentState.getStateEvents( - "m.room.member", - localUserId, - ); - - this.client.sendStateEvent( - this.room.roomId, - "m.room.member", - { - ...currentMemberState.getContent(), - [CONF_PARTICIPANT]: { - sessionId: this.sessionIds.get(localUserId), - expiresAt: new Date().getTime() + this.participantTimeout * 2, - }, - }, - localUserId, - ); - - const now = new Date().getTime(); - - // Iterate backwards so that we can remove items - for (let i = this.calls.length - 1; i >= 0; i--) { - const call = this.calls[i]; - - const opponentUserId = getCallUserId(call); - - if (!opponentUserId) { - continue; - } - - const memberStateEvent = this.room.currentState.getStateEvents( - "m.room.member", - opponentUserId, - ); - - const memberStateContent = memberStateEvent.getContent(); - - if ( - !memberStateContent || - !memberStateContent[CONF_PARTICIPANT] || - typeof memberStateContent[CONF_PARTICIPANT] !== "object" || - (memberStateContent[CONF_PARTICIPANT].expiresAt && - memberStateContent[CONF_PARTICIPANT].expiresAt < now) - ) { - this.removeCall(call, CallErrorCode.UserHangup); - } - } - - this.presenceLoopTimeout = setTimeout( - this.onPresenceLoop, - this.participantTimeout, - ); - }; - /** * Call Setup * @@ -358,48 +302,59 @@ export class GroupCall extends EventEmitter { return; } - const opponentMemberId = newCall.getOpponentMember().userId; - - logger.log(`GroupCall: incoming call from: ${opponentMemberId}`); - - const memberStateEvent = this.room.currentState.getStateEvents( - "m.room.member", - opponentMemberId, - ); - - const memberStateContent = memberStateEvent.getContent(); - - if (!memberStateContent || !memberStateContent[CONF_PARTICIPANT]) { + if (!newCall.groupCallId || newCall.groupCallId !== this.groupCallId) { newCall.reject(); return; } - const { sessionId } = memberStateContent[CONF_PARTICIPANT]; - this.sessionIds.set(opponentMemberId, sessionId); - + const opponentMemberId = newCall.getOpponentMember().userId; const existingCall = this.getCallByUserId(opponentMemberId); + logger.log(`GroupCall: incoming call from: ${opponentMemberId}`); + // Check if the user calling has an existing call and use this call instead. if (existingCall) { - this.replaceCall(existingCall, newCall, sessionId); + this.replaceCall(existingCall, newCall); } else { - this.addCall(newCall, sessionId); + this.addCall(newCall); } newCall.answerWithCallFeeds([this.localCallFeed]); }; - private onRoomStateMembers = (_event, _state, member: RoomMember) => { + /** + * Room Member State + */ + + private sendEnteredMemberStateEvent(): Promise { + return this.updateMemberState([ + { + "m.call_id": this.groupCallId, + }, + ]); + } + + private sendLeftMemberStateEvent(): Promise { + return this.updateMemberState([]); + } + + private async updateMemberState(state: any): Promise { + const localUserId = this.client.getUserId(); + + const currentStateEvent = this.room.currentState.getStateEvents("m.room.member", localUserId); + + return this.client.sendStateEvent(this.room.roomId, "m.room.member", { + ...currentStateEvent.getContent(), + [CALL_MEMBER_KEY]: state, + }, localUserId); + } + + private onMemberStateChanged = (event: MatrixEvent, state: RoomState, member: RoomMember) => { // The member events may be received for another room, which we will ignore. - if (member.roomId !== this.room.roomId) { + if (event.getRoomId() !== this.room.roomId) { return; } - logger.log(`GroupCall member state changed: ${member.userId}`); - this.onMemberChanged(member); - }; - - private onMemberChanged = (member: RoomMember) => { // Don't process your own member. const localUserId = this.client.getUserId(); @@ -407,38 +362,29 @@ export class GroupCall extends EventEmitter { return; } - // Get the latest member participant state event. - const memberStateEvent = this.room.currentState.getStateEvents( - "m.room.member", - member.userId, - ); - const memberStateContent = memberStateEvent.getContent(); + const callsState = event.getContent()[CALL_MEMBER_KEY]; - if (!memberStateContent) { + if (!callsState || !Array.isArray(callsState) || this.calls.length === 0) { return; } - const participantInfo = memberStateContent[CONF_PARTICIPANT]; + // Currently we only support a single call per room. So grab the first call. + const callState = callsState[0]; + + const callId = callState["m.call_id"]; - if (!participantInfo || typeof participantInfo !== "object") { + if (!callId) { + logger.warn(`Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`); return; } - const { expiresAt, sessionId } = participantInfo; - - // If the participant state has expired, ignore this user. - const now = new Date().getTime(); - - if (expiresAt < now) { + if (callId !== this.groupCallId) { return; } - // If there is an existing call for this member check the session id. - // If the session id changed then we can hang up the old call and start a new one. - // Otherwise, ignore the member change event because we already have an active participant. const existingCall = this.getCallByUserId(member.userId); - if (existingCall && this.sessionIds.get(member.userId) === sessionId) { + if (existingCall) { return; } @@ -448,6 +394,8 @@ export class GroupCall extends EventEmitter { return; } + logger.log(`GroupCall member state changed: ${member.userId}`); + const newCall = createNewMatrixCall( this.client, this.room.roomId, @@ -461,9 +409,9 @@ export class GroupCall extends EventEmitter { } if (existingCall) { - this.replaceCall(existingCall, newCall, sessionId); + this.replaceCall(existingCall, newCall); } else { - this.addCall(newCall, sessionId); + this.addCall(newCall); } }; @@ -475,13 +423,13 @@ export class GroupCall extends EventEmitter { return this.calls.find((call) => getCallUserId(call) === userId); } - private addCall(call: MatrixCall, sessionId: string) { + private addCall(call: MatrixCall) { this.calls.push(call); - this.initCall(call, sessionId); + this.initCall(call); this.emit(GroupCallEvent.CallsChanged, this.calls); } - private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, sessionId: string) { + private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall) { const existingCallIndex = this.calls.indexOf(existingCall); if (existingCallIndex === -1) { @@ -491,7 +439,7 @@ export class GroupCall extends EventEmitter { this.calls.splice(existingCallIndex, 1, replacementCall); this.disposeCall(existingCall, CallErrorCode.Replaced); - this.initCall(replacementCall, sessionId); + this.initCall(replacementCall); this.emit(GroupCallEvent.CallsChanged, this.calls); } @@ -510,7 +458,7 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.CallsChanged, this.calls); } - private initCall(call: MatrixCall, sessionId: string) { + private initCall(call: MatrixCall) { const opponentMemberId = getCallUserId(call); if (!opponentMemberId) { @@ -535,7 +483,6 @@ export class GroupCall extends EventEmitter { this.activeSpeakerSamples.set(opponentMemberId, Array(this.activeSpeakerSampleCount).fill( -Infinity, )); - this.sessionIds.set(opponentMemberId, sessionId); this.reEmitter.reEmit(call, Object.values(CallEvent)); } @@ -569,7 +516,6 @@ export class GroupCall extends EventEmitter { } this.activeSpeakerSamples.delete(opponentMemberId); - this.sessionIds.delete(opponentMemberId); } private onCallFeedsChanged = (call: MatrixCall) => { diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 00f99624cfb..72c81153983 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -16,9 +16,8 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { MatrixClient } from '../client'; -import { CONF_ROOM, GroupCall, IGroupCallDataChannelOptions } from "./groupCall"; +import { CALL_EVENT, GroupCall, GroupCallIntent, GroupCallType, IGroupCallDataChannelOptions } from "./groupCall"; import { RoomState } from "../models/room-state"; -import { CallType } from "./call"; import { logger } from '../logger'; export class GroupCallEventHandler { @@ -42,14 +41,6 @@ export class GroupCallEventHandler { const roomId = event.getRoomId(); const content = event.getContent(); - let callType: CallType; - - if (content.callType === "voice") { - callType = CallType.Voice; - } else { - callType = CallType.Video; - } - const room = this.client.getRoom(event.getRoomId()); if (!room) { @@ -57,6 +48,22 @@ export class GroupCallEventHandler { return; } + const groupCallId = event.getStateKey(); + + const callType = content.type; + + if (!Object.values(GroupCallType).includes(callType)) { + logger.error(`Received invalid group call type ${callType} for room ${roomId}.`); + return; + } + + const callIntent = content.intent; + + if (!Object.values(GroupCallIntent).includes(callIntent)) { + logger.error(`Received invalid group call intent ${callType} for room ${roomId}.`); + return; + } + let dataChannelOptions: IGroupCallDataChannelOptions | undefined; if (content?.dataChannelsEnabled && content?.dataChannelOptions) { @@ -69,30 +76,41 @@ export class GroupCallEventHandler { this.client, room, callType, + callIntent, content?.dataChannelsEnabled, dataChannelOptions, ); - groupCall.groupCallId = content["conf_id"]; + groupCall.groupCallId = groupCallId; return groupCall; } - private onRoomStateChanged = (_event: MatrixEvent, state: RoomState): void => { - const groupCall = this.groupCalls.get(state.roomId); - const confEvents = state.getStateEvents(CONF_ROOM); - const confEvent = confEvents.length > 0 ? confEvents[0] : null; - const content = confEvent ? confEvent.getContent() : null; - - if (groupCall && content?.type !== groupCall.type) { - // TODO: Handle the callType changing when the room state changes - logger.warn(`The group call type changed for room: ${ - state.roomId}. Changing the group call type is currently unsupported.`); - } if (groupCall && !content?.active) { - groupCall.endCall(false); - } else if (!groupCall && content?.active) { - const groupCall = this.createGroupCallFromRoomStateEvent(confEvent); + private onRoomStateChanged = (event: MatrixEvent, state: RoomState): void => { + if (event.getType() !== CALL_EVENT) { + return; + } + + const groupCallId = event.getStateKey(); + const content = event.getContent(); + + const currentGroupCall = this.groupCalls.get(state.roomId); + + if (!currentGroupCall && !content["m.terminated"]) { + const groupCall = this.createGroupCallFromRoomStateEvent(event); this.groupCalls.set(state.roomId, groupCall); this.client.emit("GroupCall.incoming", groupCall); + } else if (currentGroupCall && currentGroupCall.groupCallId === groupCallId) { + if (content["m.terminated"]) { + currentGroupCall.terminate(false); + } else if (content["m.type"] !== currentGroupCall.type) { + // TODO: Handle the callType changing when the room state changes + logger.warn(`The group call type changed for room: ${ + state.roomId}. Changing the group call type is currently unsupported.`); + } + } else if (currentGroupCall && currentGroupCall.groupCallId !== groupCallId) { + // TODO: Handle new group calls and multiple group calls + logger.warn(`Multiple group calls detected for room: ${ + state.roomId}. Multiple group calls are currently unsupported.`); } }; } From ce0b0ea182d7f15a643e982098c63f1c5d2c6b41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Sep 2021 08:39:54 +0200 Subject: [PATCH 052/291] De-duplicate invite/answer code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 138 ++++++++++----------------------------------- 1 file changed, 31 insertions(+), 107 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index dfddbaca8e2..2a8fe1b15e2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -796,8 +796,6 @@ export class MatrixCall extends EventEmitter { // TODO: Figure out how to do this if (audio === false && video === false) throw new Error("You CANNOT answer a call without media"); - logger.debug(`Answering call ${this.callId}`); - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { this.setState(CallState.WaitLocalMedia); this.waitForLocalAVStream = true; @@ -808,7 +806,16 @@ export class MatrixCall extends EventEmitter { this.shouldAnswerWithMediaType(video, this.hasRemoteUserMediaVideoTrack, "video"), ); this.waitForLocalAVStream = false; - this.gotUserMediaForAnswer(mediaStream); + const callFeed = new CallFeed( + mediaStream, + this.client.getUserId(), + SDPStreamMetadataPurpose.Usermedia, + this.client, + this.roomId, + audio, + video, + ); + this.answerWithCallFeeds([callFeed], true); } catch (e) { this.getUserMediaFailed(e); return; @@ -818,26 +825,14 @@ export class MatrixCall extends EventEmitter { } } - public answerWithCallFeeds(callFeeds: CallFeed[]): void { - this.stopLocalMediaOnEnd = false; + public answerWithCallFeeds(callFeeds: CallFeed[], stopLocalMediaOnEnd?: boolean): void { if (this.inviteOrAnswerSent) return; logger.debug(`Answering call ${this.callId}`); - if (!this.localUsermediaStream && !this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - this.waitForLocalAVStream = true; + this.stopLocalMediaOnEnd = stopLocalMediaOnEnd; - try { - this.waitForLocalAVStream = false; - this.gotCallFeedsForAnswer(callFeeds); - } catch (e) { - this.getUserMediaFailed(e); - return; - } - } else if (this.waitForLocalAVStream) { - this.setState(CallState.WaitLocalMedia); - } + this.gotCallFeedsForAnswer(callFeeds); } /** @@ -851,11 +846,7 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { logger.debug("Handing local stream to new call"); - if (this.stopLocalMediaOnEnd) { - newCall.gotUserMediaForAnswer(this.localUsermediaStream); - } else { - newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); - } + newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -1187,27 +1178,6 @@ export class MatrixCall extends EventEmitter { setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); } - /** - * Internal - * @param {Object} stream - */ - private gotUserMediaForInvite = async (stream: MediaStream): Promise => { - if (this.successor) { - this.successor.gotUserMediaForAnswer(stream); - return; - } - if (this.callHasEnded()) { - this.stopAllMedia(); - return; - } - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - this.setState(CallState.CreateOffer); - - logger.debug("gotUserMediaForInvite"); - // Now we wait for the negotiationneeded event - }; - private gotCallFeedsForInvite(callFeeds: CallFeed[]): void { if (this.successor) { this.successor.gotCallFeedsForAnswer(callFeeds); @@ -1274,41 +1244,6 @@ export class MatrixCall extends EventEmitter { this.sendCandidateQueue(); } - private gotUserMediaForAnswer = async (stream: MediaStream): Promise => { - if (this.callHasEnded()) { - return; - } - - this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia); - this.setState(CallState.CreateAnswer); - - let myAnswer; - try { - this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); - this.setState(CallState.Connecting); - - // Allow a short time for initial candidates to be gathered - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - this.sendAnswer(); - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; - } - }; - private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { if (this.callHasEnded()) return; @@ -2112,30 +2047,23 @@ export class MatrixCall extends EventEmitter { * @throws if have passed audio=false. */ public async placeCall(audio: boolean, video: boolean): Promise { - logger.debug(`placeCall audio=${audio} video=${video}`); if (!audio) { throw new Error("You CANNOT start a call without audio"); } - this.checkForErrorListener(); - // XXX Find a better way to do this - this.client.callEventHandler.calls.set(this.callId, this); this.setState(CallState.WaitLocalMedia); - this.direction = CallDirection.Outbound; - - // make sure we have valid turn creds. Unless something's gone wrong, it should - // poll and keep the credentials valid so this should be instant. - const haveTurnCreds = await this.client.checkTurnServers(); - if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); - } - - // create the peer connection now so it can be gathering candidates while we get user - // media (assuming a candidate pool size is configured) - this.peerConn = this.createPeerConnection(); try { const mediaStream = await this.client.getMediaHandler().getUserMediaStream(audio, video); - this.gotUserMediaForInvite(mediaStream); + const callFeed = new CallFeed( + mediaStream, + this.client.getUserId(), + SDPStreamMetadataPurpose.Usermedia, + this.client, + this.roomId, + false, + false, + ); + await this.placeCallWithCallFeeds([callFeed], true); } catch (e) { this.getUserMediaFailed(e); return; @@ -2148,13 +2076,15 @@ export class MatrixCall extends EventEmitter { * @throws if you have not specified a listener for 'error' events. * @throws if have passed audio=false. */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise { - this.stopLocalMediaOnEnd = false; + public async placeCallWithCallFeeds(callFeeds: CallFeed[], stopLocalMediaOnEnd?: boolean): Promise { + logger.debug("Placing call with", callFeeds); + this.checkForErrorListener(); + this.stopLocalMediaOnEnd = stopLocalMediaOnEnd; + this.direction = CallDirection.Outbound; + // XXX Find a better way to do this this.client.callEventHandler.calls.set(this.callId, this); - this.setState(CallState.WaitLocalMedia); - this.direction = CallDirection.Outbound; // make sure we have valid turn creds. Unless something's gone wrong, it should // poll and keep the credentials valid so this should be instant. @@ -2166,13 +2096,7 @@ export class MatrixCall extends EventEmitter { // create the peer connection now so it can be gathering candidates while we get user // media (assuming a candidate pool size is configured) this.peerConn = this.createPeerConnection(); - - try { - this.gotCallFeedsForInvite(callFeeds); - } catch (e) { - this.getUserMediaFailed(e); - return; - } + this.gotCallFeedsForInvite(callFeeds); } private createPeerConnection(): RTCPeerConnection { From 5251dcf67fc248ee29baa36129ac28a22587930c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Sep 2021 08:53:44 +0200 Subject: [PATCH 053/291] De-duplicate pushNewLocalFeed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 33 ++++++--------------------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2a8fe1b15e2..2ef90d3f9d5 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -570,40 +570,19 @@ export class MatrixCall extends EventEmitter { private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { const userId = this.client.getUserId(); + // TODO: Find out what is going on here + // why do we enable audio (and only audio) tracks here? -- matthew + setTracksEnabled(stream.getAudioTracks(), true); + // We try to replace an existing feed if there already is one with the same purpose const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); if (existingFeed) { existingFeed.setNewStream(stream); } else { - this.feeds.push(new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false)); + const callFeed = new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false); this.emit(CallEvent.FeedsChanged, this.feeds); + this.pushLocalFeed(callFeed, addToPeerConnection); } - - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew - setTracksEnabled(stream.getAudioTracks(), true); - - if (addToPeerConnection) { - const senderArray = purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - - this.emit(CallEvent.FeedsChanged, this.feeds); - for (const track of stream.getTracks()) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${purpose}"` + - `) to peer connection`, - ); - senderArray.push(this.peerConn.addTrack(track, stream)); - } - } - - logger.info(`Pushed local stream (id="${stream.id}", active="${stream.active}", purpose="${purpose}")`); } /** From 96bde1f706590947eaa80640247bef3dc655ee37 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Sep 2021 18:17:36 +0200 Subject: [PATCH 054/291] Fix field keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCallEventHandler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 72c81153983..04f5053d900 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -50,14 +50,14 @@ export class GroupCallEventHandler { const groupCallId = event.getStateKey(); - const callType = content.type; + const callType = content["m.type"]; if (!Object.values(GroupCallType).includes(callType)) { logger.error(`Received invalid group call type ${callType} for room ${roomId}.`); return; } - const callIntent = content.intent; + const callIntent = content["m.intent"]; if (!Object.values(GroupCallIntent).includes(callIntent)) { logger.error(`Received invalid group call intent ${callType} for room ${roomId}.`); From 73cb5e1ee9d47b42d80b4031180db7b54eb43384 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Sep 2021 19:10:42 +0200 Subject: [PATCH 055/291] Fix order of execution MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2ef90d3f9d5..44a1388ecd1 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -580,8 +580,8 @@ export class MatrixCall extends EventEmitter { existingFeed.setNewStream(stream); } else { const callFeed = new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false); - this.emit(CallEvent.FeedsChanged, this.feeds); this.pushLocalFeed(callFeed, addToPeerConnection); + this.emit(CallEvent.FeedsChanged, this.feeds); } } From 1a824750dd78a3269355a0de5bc506a1d9a139f2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 28 Sep 2021 19:11:53 +0200 Subject: [PATCH 056/291] Don't emit the same thing twice MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 44a1388ecd1..24719fc5168 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -581,7 +581,6 @@ export class MatrixCall extends EventEmitter { } else { const callFeed = new CallFeed(stream, userId, purpose, this.client, this.roomId, false, false); this.pushLocalFeed(callFeed, addToPeerConnection); - this.emit(CallEvent.FeedsChanged, this.feeds); } } From f627507b86b6512a213d7d48c627f9b36a48359c Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 11:03:42 -0700 Subject: [PATCH 057/291] Test prepare script for git installs --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index d48525d8382..c14aafeb783 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "13.0.0", "description": "Matrix Client-Server SDK for Javascript", "scripts": { - "prepublishOnly": "yarn build", + "prepare": "yarn build", "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"", "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build", "clean": "rimraf lib dist", From 5111ca622af03ce0fe04606dc12ed97abd15fa07 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 11:36:11 -0700 Subject: [PATCH 058/291] Change main path --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c14aafeb783..051830143fe 100644 --- a/package.json +++ b/package.json @@ -29,7 +29,7 @@ "keywords": [ "matrix-org" ], - "main": "./src/index.ts", + "main": "./lib/index.js", "browser": "./lib/browser-index.js", "matrix_src_main": "./src/index.ts", "matrix_src_browser": "./src/browser-index.js", From 529d61b5f4829fbd8ef5fcdc2501897782ad5267 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 11:44:12 -0700 Subject: [PATCH 059/291] Add .npmignore --- .npmignore | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 .npmignore diff --git a/.npmignore b/.npmignore new file mode 100644 index 00000000000..e69de29bb2d From 5da072712debddcf8e32aadfe3772ece98654467 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 11:47:09 -0700 Subject: [PATCH 060/291] Use prepack instead of prepare --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 051830143fe..3550b421b70 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "13.0.0", "description": "Matrix Client-Server SDK for Javascript", "scripts": { - "prepare": "yarn build", + "prepack": "yarn build", "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"", "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build", "clean": "rimraf lib dist", From 1e65bfd316be75306c597cd4f1a3505113d29cb1 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 14:35:32 -0700 Subject: [PATCH 061/291] Fix initLocalCallFeed state --- src/webrtc/groupCall.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e2cc66d261e..740ae4e3610 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -135,6 +135,8 @@ export class GroupCall extends EventEmitter { this.localCallFeed = callFeed; this.addUserMediaFeed(callFeed); + this.setState(GroupCallState.LocalCallFeedInitialized); + return callFeed; } From 137fd2bd4089a05b364f7b4f3bcfa3c0ee569e31 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 14:35:42 -0700 Subject: [PATCH 062/291] Export group call enums --- src/matrix.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/matrix.ts b/src/matrix.ts index 581236ee99d..ccdfea69ed0 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -48,6 +48,12 @@ export * from "./crypto/store/indexeddb-crypto-store"; export * from "./content-repo"; export * as ContentHelpers from "./content-helpers"; export { createNewMatrixCall } from "./webrtc/call"; +export { + GroupCallEvent, + GroupCallIntent, + GroupCallState, + GroupCallType, +} from "./webrtc/groupCall"; // expose the underlying request object so different environments can use // different request libs (e.g. request or browser-request) From 625983a2b229545d4b0a57352775b389a2d06ad4 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 14:37:07 -0700 Subject: [PATCH 063/291] Revert changes to package.json and .npmignore --- .npmignore | 0 package.json | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 .npmignore diff --git a/.npmignore b/.npmignore deleted file mode 100644 index e69de29bb2d..00000000000 diff --git a/package.json b/package.json index 3550b421b70..d48525d8382 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "version": "13.0.0", "description": "Matrix Client-Server SDK for Javascript", "scripts": { - "prepack": "yarn build", + "prepublishOnly": "yarn build", "start": "echo THIS IS FOR LEGACY PURPOSES ONLY. && babel src -w -s -d lib --verbose --extensions \".ts,.js\"", "dist": "echo 'This is for the release script so it can make assets (browser bundle).' && yarn build", "clean": "rimraf lib dist", @@ -29,7 +29,7 @@ "keywords": [ "matrix-org" ], - "main": "./lib/index.js", + "main": "./src/index.ts", "browser": "./lib/browser-index.js", "matrix_src_main": "./src/index.ts", "matrix_src_browser": "./src/browser-index.js", From 3f4522ba88a4b60637ba77f4c91e7458780e1946 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 16:22:44 -0700 Subject: [PATCH 064/291] Add more verbose logging for testing. Create group calls on first sync. --- src/client.ts | 32 +----------------------- src/webrtc/callEventHandler.ts | 8 ++++++ src/webrtc/groupCall.ts | 38 +++++++++++++++++------------ src/webrtc/groupCallEventHandler.ts | 33 ++++++++++++++++++++++--- 4 files changed, 62 insertions(+), 49 deletions(-) diff --git a/src/client.ts b/src/client.ts index 027d105fb2d..b9ec0f5bfdd 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1343,37 +1343,7 @@ export class MatrixClient extends EventEmitter { * @returns {GroupCall} The group call or null if it doesn't already exist. */ public getGroupCallForRoom(roomId: string): GroupCall | null { - const groupCall = this.groupCallEventHandler.groupCalls.get(roomId); - - if (groupCall) { - return groupCall; - } - - const room = this.getRoom(roomId); - - if (!room) { - return null; - } - - const callEvents = room.currentState.getStateEvents(CALL_EVENT); - - if (callEvents.length === 0) { - return null; - } - - const sortedCallEvents = callEvents.sort((a, b) => a.getTs() - b.getTs()); - - for (const callEvent of sortedCallEvents) { - const content = callEvent.getContent(); - - if (content["m.terminated"]) { - continue; - } - - return this.groupCallEventHandler.createGroupCallFromRoomStateEvent(callEvent); - } - - return null; + return this.groupCallEventHandler.groupCalls.get(roomId) || null; } /** diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 377b5fe646d..c1bbaafab15 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -131,6 +131,7 @@ export class CallEventHandler { }; private onToDeviceEvent = (event: MatrixEvent): void => { + logger.log("onToDeviceEvent", event); if (!this.eventIsACall(event)) return; this.toDeviceCallEventBuffer.push(event); @@ -139,6 +140,8 @@ export class CallEventHandler { private async evaluateToDeviceEventBuffer(): Promise { if (this.client.getSyncState() !== SyncState.Syncing) return; + logger.log("processing to device events"); + for (const event of this.toDeviceCallEventBuffer) { try { await this.handleCallEvent(event, true); @@ -146,6 +149,9 @@ export class CallEventHandler { logger.error("Caught exception handling call event", e); } } + + logger.log("processing to device events finished"); + this.toDeviceCallEventBuffer = []; } @@ -170,6 +176,8 @@ export class CallEventHandler { let call = content.call_id ? this.calls.get(content.call_id) : undefined; //console.info("RECV %s content=%s", type, JSON.stringify(content)); + logger.log("handleCallEvent", callRoomId, groupCallId, type, event); + if (!callRoomId) return; if (type === EventType.CallInvite) { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 740ae4e3610..2a3ec79eb2a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -150,19 +150,31 @@ export class GroupCall extends EventEmitter { await this.initLocalCallFeed(); } + logger.log(`Sending member state event with current call.`); + this.sendEnteredMemberStateEvent(); this.activeSpeaker = this.client.getUserId(); this.setState(GroupCallState.Entered); - this.processInitialCalls(); + logger.log(`Entered group call ${this.groupCallId}`); + + logger.log("processing initial calls"); + + const calls = this.client.callEventHandler.calls.values(); + + for (const call of calls) { + this.onIncomingCall(call); + } // Set up participants for the members currently in the room. // Other members will be picked up by the RoomState.members event. const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents("m.room.member"); + logger.log("Processing initial members"); + for (const stateEvent of memberStateEvents) { const member = this.room.getMember(stateEvent.getStateKey()); this.onMemberStateChanged(stateEvent, roomState, member); @@ -285,14 +297,6 @@ export class GroupCall extends EventEmitter { * as they are observed by the RoomState.members event. */ - private processInitialCalls() { - const calls = this.client.callEventHandler.calls.values(); - - for (const call of calls) { - this.onIncomingCall(call); - } - } - private onIncomingCall = (newCall: MatrixCall) => { // The incoming calls may be for another room, which we will ignore. if (newCall.roomId !== this.room.roomId) { @@ -305,6 +309,8 @@ export class GroupCall extends EventEmitter { } if (!newCall.groupCallId || newCall.groupCallId !== this.groupCallId) { + logger.log(`Incoming call with groupCallId ${ + newCall.groupCallId} ignored because it doesn't match the current group call`); newCall.reject(); return; } @@ -329,7 +335,7 @@ export class GroupCall extends EventEmitter { */ private sendEnteredMemberStateEvent(): Promise { - return this.updateMemberState([ + return this.updateMemberCallsState([ { "m.call_id": this.groupCallId, }, @@ -337,10 +343,10 @@ export class GroupCall extends EventEmitter { } private sendLeftMemberStateEvent(): Promise { - return this.updateMemberState([]); + return this.updateMemberCallsState([]); } - private async updateMemberState(state: any): Promise { + private async updateMemberCallsState(state: any): Promise { const localUserId = this.client.getUserId(); const currentStateEvent = this.room.currentState.getStateEvents("m.room.member", localUserId); @@ -366,7 +372,8 @@ export class GroupCall extends EventEmitter { const callsState = event.getContent()[CALL_MEMBER_KEY]; - if (!callsState || !Array.isArray(callsState) || this.calls.length === 0) { + if (!callsState || !Array.isArray(callsState) || callsState.length === 0) { + logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); return; } @@ -381,6 +388,7 @@ export class GroupCall extends EventEmitter { } if (callId !== this.groupCallId) { + logger.log(`Call id does not match group call id, ignoring.`); return; } @@ -393,11 +401,10 @@ export class GroupCall extends EventEmitter { // Only initiate a call with a user who has a userId that is lexicographically // less than your own. Otherwise, that user will call you. if (member.userId < localUserId) { + logger.log(`Waiting for ${member.userId} to send call invite.`); return; } - logger.log(`GroupCall member state changed: ${member.userId}`); - const newCall = createNewMatrixCall( this.client, this.room.roomId, @@ -410,6 +417,7 @@ export class GroupCall extends EventEmitter { newCall.createDataChannel("datachannel", this.dataChannelOptions); } + // TODO: This existingCall code path is never reached, do we still need it? if (existingCall) { this.replaceCall(existingCall, newCall); } else { diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 04f5053d900..385d22552d9 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -26,6 +26,28 @@ export class GroupCallEventHandler { constructor(private client: MatrixClient) { } public start(): void { + const rooms = this.client.getRooms(); + + for (const room of rooms) { + const callEvents = room.currentState.getStateEvents(CALL_EVENT); + const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); + + for (const callEvent of sortedCallEvents) { + const content = callEvent.getContent(); + + if (content["m.terminated"]) { + continue; + } + + const groupCall = this.createGroupCallFromRoomStateEvent(callEvent); + + if (groupCall) { + this.groupCalls.set(room.roomId, groupCall); + this.client.emit("GroupCall.incoming", groupCall); + } + } + } + this.client.on("RoomState.events", this.onRoomStateChanged); } @@ -41,7 +63,9 @@ export class GroupCallEventHandler { const roomId = event.getRoomId(); const content = event.getContent(); - const room = this.client.getRoom(event.getRoomId()); + logger.log("createGroupCallFromRoomStateEvent", roomId); + + const room = this.client.getRoom(roomId); if (!room) { logger.error(`Couldn't find room ${roomId} for GroupCall`); @@ -97,8 +121,11 @@ export class GroupCallEventHandler { if (!currentGroupCall && !content["m.terminated"]) { const groupCall = this.createGroupCallFromRoomStateEvent(event); - this.groupCalls.set(state.roomId, groupCall); - this.client.emit("GroupCall.incoming", groupCall); + + if (groupCall) { + this.groupCalls.set(state.roomId, groupCall); + this.client.emit("GroupCall.incoming", groupCall); + } } else if (currentGroupCall && currentGroupCall.groupCallId === groupCallId) { if (content["m.terminated"]) { currentGroupCall.terminate(false); From 688327dab5dece970f15426bb8fb010f09de6f6f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 17:00:17 -0700 Subject: [PATCH 065/291] Handle new group call rooms --- src/webrtc/groupCallEventHandler.ts | 51 +++++++++++++++-------------- 1 file changed, 27 insertions(+), 24 deletions(-) diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 385d22552d9..657727203ad 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -17,6 +17,7 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { MatrixClient } from '../client'; import { CALL_EVENT, GroupCall, GroupCallIntent, GroupCallType, IGroupCallDataChannelOptions } from "./groupCall"; +import { Room } from "../models/room"; import { RoomState } from "../models/room-state"; import { logger } from '../logger'; @@ -29,25 +30,10 @@ export class GroupCallEventHandler { const rooms = this.client.getRooms(); for (const room of rooms) { - const callEvents = room.currentState.getStateEvents(CALL_EVENT); - const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); - - for (const callEvent of sortedCallEvents) { - const content = callEvent.getContent(); - - if (content["m.terminated"]) { - continue; - } - - const groupCall = this.createGroupCallFromRoomStateEvent(callEvent); - - if (groupCall) { - this.groupCalls.set(room.roomId, groupCall); - this.client.emit("GroupCall.incoming", groupCall); - } - } + this.createGroupCallForRoom(room); } + this.client.on("Room", this.onRoomsChanged); this.client.on("RoomState.events", this.onRoomStateChanged); } @@ -59,7 +45,22 @@ export class GroupCallEventHandler { return [...this.groupCalls.values()].find((groupCall) => groupCall.groupCallId === groupCallId); } - public createGroupCallFromRoomStateEvent(event: MatrixEvent) { + private createGroupCallForRoom(room: Room): GroupCall | undefined { + const callEvents = room.currentState.getStateEvents(CALL_EVENT); + const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); + + for (const callEvent of sortedCallEvents) { + const content = callEvent.getContent(); + + if (content["m.terminated"]) { + continue; + } + + return this.createGroupCallFromRoomStateEvent(callEvent); + } + } + + private createGroupCallFromRoomStateEvent(event: MatrixEvent): GroupCall | undefined { const roomId = event.getRoomId(); const content = event.getContent(); @@ -106,9 +107,16 @@ export class GroupCallEventHandler { ); groupCall.groupCallId = groupCallId; + this.groupCalls.set(room.roomId, groupCall); + this.client.emit("GroupCall.incoming", groupCall); + return groupCall; } + private onRoomsChanged = (room: Room) => { + this.createGroupCallForRoom(room); + }; + private onRoomStateChanged = (event: MatrixEvent, state: RoomState): void => { if (event.getType() !== CALL_EVENT) { return; @@ -120,12 +128,7 @@ export class GroupCallEventHandler { const currentGroupCall = this.groupCalls.get(state.roomId); if (!currentGroupCall && !content["m.terminated"]) { - const groupCall = this.createGroupCallFromRoomStateEvent(event); - - if (groupCall) { - this.groupCalls.set(state.roomId, groupCall); - this.client.emit("GroupCall.incoming", groupCall); - } + this.createGroupCallFromRoomStateEvent(event); } else if (currentGroupCall && currentGroupCall.groupCallId === groupCallId) { if (content["m.terminated"]) { currentGroupCall.terminate(false); From 326aec9f9ea7b29c6c07bbcc84dc5288300abc58 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 29 Sep 2021 17:30:34 -0700 Subject: [PATCH 066/291] Fix getUserMediaStream --- src/webrtc/mediaHandler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 2792f1e180f..b2531bc48ef 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -53,6 +53,7 @@ export class MediaHandler { const matchingStream = this.userMediaStreams.find((stream) => { if (audio !== (stream.getAudioTracks().length > 0)) return false; if (video !== (stream.getVideoTracks().length > 0)) return false; + return true; }); if (matchingStream) { From bb504bc0014bc855304b2913fc0a508dbb83bee0 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 30 Sep 2021 11:39:34 -0700 Subject: [PATCH 067/291] Handle getUserMedia permissions blocked --- src/webrtc/groupCall.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 2a3ec79eb2a..715ffd7c6dc 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -115,7 +115,14 @@ export class GroupCall extends EventEmitter { this.setState(GroupCallState.InitializingLocalCallFeed); - const stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); + let stream: MediaStream; + + try { + stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); + } catch (error) { + this.setState(GroupCallState.LocalCallFeedUninitialized); + throw error; + } const userId = this.client.getUserId(); From 0d964523a929509549a553912de69a3252d965f3 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 30 Sep 2021 16:04:15 -0700 Subject: [PATCH 068/291] Add screensharing to GroupCall --- src/webrtc/groupCall.ts | 176 ++++++++++++++++++++++++++++++++++--- src/webrtc/mediaHandler.ts | 2 +- 2 files changed, 167 insertions(+), 11 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 715ffd7c6dc..e7e043959bc 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -35,7 +35,25 @@ export enum GroupCallEvent { ActiveSpeakerChanged = "active_speaker_changed", CallsChanged = "calls_changed", UserMediaFeedsChanged = "user_media_feeds_changed", + ScreenshareFeedsChanged = "screenshare_feeds_changed", + LocalScreenshareStateChanged = "local_screenshare_state_changed", LocalMuteStateChanged = "local_mute_state_changed", + Error = "error" +} + +export enum GroupCallErrorCode { + NoUserMedia = "no_user_media" +} + +export class GroupCallError extends Error { + code: string; + + constructor(code: GroupCallErrorCode, msg: string, err: Error) { + // Still don't think there's any way to have proper nested errors + super(msg + ": " + err); + + this.code = code; + } } export interface IGroupCallDataChannelOptions { @@ -77,10 +95,13 @@ export class GroupCall extends EventEmitter { public participantTimeout = 1000 * 15; public state = GroupCallState.LocalCallFeedUninitialized; - public activeSpeaker: string; // userId - public localCallFeed: CallFeed; + public activeSpeaker?: string; // userId + public localCallFeed?: CallFeed; + public localScreenshareFeed?: CallFeed; + public localDesktopCapturerSourceId?: string; public calls: MatrixCall[] = []; public userMediaFeeds: CallFeed[] = []; + public screenshareFeeds: CallFeed[] = []; public groupCallId: string; private userMediaFeedHandlers: Map = new Map(); @@ -199,6 +220,13 @@ export class GroupCall extends EventEmitter { this.localCallFeed = null; } + if (this.localScreenshareFeed) { + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + this.removeScreenshareFeed(this.localScreenshareFeed); + this.localScreenshareFeed = undefined; + this.localDesktopCapturerSourceId = undefined; + } + this.client.getMediaHandler().stopAllStreams(); if (this.state !== GroupCallState.Entered) { @@ -295,6 +323,70 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); } + public async setScreensharingEnabled( + enabled: boolean, desktopCapturerSourceId?: string, + ): Promise { + if (enabled === this.isScreensharing()) { + return enabled; + } + + if (enabled) { + try { + logger.log("Asking for screensharing permissions..."); + + const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + + logger.log("Screensharing permissions granted. Setting screensharing enabled on all calls"); + + const callFeed = new CallFeed( + stream, + this.client.getUserId(), + SDPStreamMetadataPurpose.Screenshare, + this.client, + this.room.roomId, + false, + false, + ); + + this.localScreenshareFeed = callFeed; + this.localDesktopCapturerSourceId = desktopCapturerSourceId; + this.addScreenshareFeed(callFeed); + + this.emit( + GroupCallEvent.LocalScreenshareStateChanged, + true, + this.localScreenshareFeed, + this.localDesktopCapturerSourceId, + ); + + // TODO: handle errors + await Promise.all(this.calls.map(call => call.setScreensharingEnabled(true, desktopCapturerSourceId))); + + logger.log("screensharing enabled on all calls"); + + return true; + } catch (error) { + logger.error("enabling screensharing error", error); + this.emit(GroupCallEvent.Error, + new GroupCallError(GroupCallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", error), + ); + return false; + } + } else { + await Promise.all(this.calls.map(call => call.setScreensharingEnabled(false, desktopCapturerSourceId))); + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + this.removeScreenshareFeed(this.localScreenshareFeed); + this.localScreenshareFeed = undefined; + this.localDesktopCapturerSourceId = undefined; + this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); + return false; + } + } + + public isScreensharing(): boolean { + return !!this.localScreenshareFeed; + } + /** * Call Setup * @@ -497,6 +589,10 @@ export class GroupCall extends EventEmitter { call.on(CallEvent.State, onCallStateChanged); call.on(CallEvent.Hangup, onCallHangup); + if (this.isScreensharing()) { + call.setScreensharingEnabled(true, this.localDesktopCapturerSourceId); + } + this.activeSpeakerSamples.set(opponentMemberId, Array(this.activeSpeakerSampleCount).fill( -Infinity, )); @@ -532,6 +628,12 @@ export class GroupCall extends EventEmitter { this.removeUserMediaFeed(usermediaFeed); } + const screenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); + + if (screenshareFeed) { + this.removeScreenshareFeed(screenshareFeed); + } + this.activeSpeakerSamples.delete(opponentMemberId); } @@ -546,16 +648,28 @@ export class GroupCall extends EventEmitter { const remoteUsermediaFeed = call.remoteUsermediaFeed; const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; - if (!remoteFeedChanged) { - return; + if (remoteFeedChanged) { + if (!currentUserMediaFeed && remoteUsermediaFeed) { + this.addUserMediaFeed(remoteUsermediaFeed); + } else if (currentUserMediaFeed && remoteUsermediaFeed) { + this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); + } else if (currentUserMediaFeed && !remoteUsermediaFeed) { + this.removeUserMediaFeed(currentUserMediaFeed); + } } - if (!currentUserMediaFeed && remoteUsermediaFeed) { - this.addUserMediaFeed(remoteUsermediaFeed); - } else if (currentUserMediaFeed && remoteUsermediaFeed) { - this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); - } else if (currentUserMediaFeed && !remoteUsermediaFeed) { - this.removeUserMediaFeed(currentUserMediaFeed); + const currentScreenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); + const remoteScreensharingFeed = call.remoteScreensharingFeed; + const remoteScreenshareFeedChanged = remoteScreensharingFeed !== currentScreenshareFeed; + + if (remoteScreenshareFeedChanged) { + if (!currentScreenshareFeed && remoteScreensharingFeed) { + this.addScreenshareFeed(remoteScreensharingFeed); + } else if (currentScreenshareFeed && remoteScreensharingFeed) { + this.replaceScreenshareFeed(currentScreenshareFeed, remoteScreensharingFeed); + } else if (currentScreenshareFeed && !remoteScreensharingFeed) { + this.removeScreenshareFeed(currentScreenshareFeed); + } } }; @@ -705,4 +819,46 @@ export class GroupCall extends EventEmitter { this.activeSpeakerInterval, ); }; + + /** + * Screenshare Call Feed Event Handlers + */ + + public getScreenshareFeedByUserId(userId: string) { + return this.screenshareFeeds.find((feed) => feed.userId === userId); + } + + private addScreenshareFeed(callFeed: CallFeed) { + logger.log("added screenshare feed"); + this.screenshareFeeds.push(callFeed); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } + + private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { + logger.log("replaced screenshare feed"); + const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === existingFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find screenshare feed to replace"); + } + + this.screenshareFeeds.splice(feedIndex, 1, replacementFeed); + + existingFeed.dispose(); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } + + private removeScreenshareFeed(callFeed: CallFeed) { + logger.log("removed screenshare feed"); + const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === callFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find screenshare feed to remove"); + } + + this.screenshareFeeds.splice(feedIndex, 1); + + callFeed.dispose(); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } } diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index b2531bc48ef..3f3dcaad62a 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -90,7 +90,7 @@ export class MediaHandler { /** * @returns {MediaStream} based on passed parameters */ - public async getScreensharingStream(desktopCapturerSourceId: string): Promise { + public async getScreensharingStream(desktopCapturerSourceId?: string): Promise { let stream: MediaStream; if (this.screensharingStreams.length === 0) { From 8be4ca909e44d5c42505a7e22136ad3a853fcab4 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 4 Oct 2021 15:36:36 -0700 Subject: [PATCH 069/291] Add participants to GroupCall --- src/webrtc/groupCall.ts | 63 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 57 insertions(+), 6 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e7e043959bc..853ec6329a6 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -38,6 +38,7 @@ export enum GroupCallEvent { ScreenshareFeedsChanged = "screenshare_feeds_changed", LocalScreenshareStateChanged = "local_screenshare_state_changed", LocalMuteStateChanged = "local_mute_state_changed", + ParticipantsChanged = "participants_changed", Error = "error" } @@ -100,6 +101,7 @@ export class GroupCall extends EventEmitter { public localScreenshareFeed?: CallFeed; public localDesktopCapturerSourceId?: string; public calls: MatrixCall[] = []; + public participants: RoomMember[] = []; public userMediaFeeds: CallFeed[] = []; public screenshareFeeds: CallFeed[] = []; public groupCallId: string; @@ -121,6 +123,18 @@ export class GroupCall extends EventEmitter { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = genCallID(); + + const roomState = this.room.currentState; + const memberStateEvents = roomState.getStateEvents("m.room.member"); + + logger.log("Processing initial members", memberStateEvents); + + for (const stateEvent of memberStateEvents) { + const member = this.room.getMember(stateEvent.getStateKey()); + this.onMemberStateChanged(stateEvent, roomState, member); + } + + this.client.on("RoomState.members", this.onMemberStateChanged); } private setState(newState: GroupCallState): void { @@ -208,7 +222,6 @@ export class GroupCall extends EventEmitter { this.onMemberStateChanged(stateEvent, roomState, member); } - this.client.on("RoomState.members", this.onMemberStateChanged); this.client.on("Call.incoming", this.onIncomingCall); this.onActiveSpeakerLoop(); @@ -241,11 +254,6 @@ export class GroupCall extends EventEmitter { this.activeSpeaker = null; clearTimeout(this.activeSpeakerLoopTimeout); - - this.client.removeListener( - "RoomState.members", - this.onMemberStateChanged, - ); this.client.removeListener("Call.incoming", this.onIncomingCall); } @@ -257,6 +265,12 @@ export class GroupCall extends EventEmitter { public async terminate(emitStateEvent = true) { this.dispose(); + this.participants = []; + this.client.removeListener( + "RoomState.members", + this.onMemberStateChanged, + ); + this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); if (emitStateEvent) { @@ -473,6 +487,7 @@ export class GroupCall extends EventEmitter { if (!callsState || !Array.isArray(callsState) || callsState.length === 0) { logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); + this.removeParticipant(member); return; } @@ -483,11 +498,19 @@ export class GroupCall extends EventEmitter { if (!callId) { logger.warn(`Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`); + this.removeParticipant(member); return; } if (callId !== this.groupCallId) { logger.log(`Call id does not match group call id, ignoring.`); + this.removeParticipant(member); + return; + } + + this.addParticipant(member); + + if (this.state !== GroupCallState.Entered) { return; } @@ -861,4 +884,32 @@ export class GroupCall extends EventEmitter { callFeed.dispose(); this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } + + /** + * Participant Management + */ + + private addParticipant(member: RoomMember) { + if (this.participants.find((m) => m.userId === member.userId)) { + return; + } + + this.participants.push(member); + + this.emit(GroupCallEvent.ParticipantsChanged, this.participants); + this.client.emit("GroupCall.participants", this.participants, this); + } + + private removeParticipant(member: RoomMember) { + const index = this.participants.findIndex((m) => m.userId === member.userId); + + if (index === -1) { + return; + } + + this.participants.splice(index, 1); + + this.emit(GroupCallEvent.ParticipantsChanged, this.participants); + this.client.emit("GroupCall.participants", this.participants, this); + } } From ba57736bf6d6d99342c64cca5eb6156ee7b9e178 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 4 Oct 2021 15:43:56 -0700 Subject: [PATCH 070/291] Remove log that's stalling FF --- src/webrtc/call.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 24719fc5168..b6127a11c24 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2055,8 +2055,6 @@ export class MatrixCall extends EventEmitter { * @throws if have passed audio=false. */ public async placeCallWithCallFeeds(callFeeds: CallFeed[], stopLocalMediaOnEnd?: boolean): Promise { - logger.debug("Placing call with", callFeeds); - this.checkForErrorListener(); this.stopLocalMediaOnEnd = stopLocalMediaOnEnd; this.direction = CallDirection.Outbound; From 8ac3841a2f5c54f02fb1fd39c5895213c3802770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 3 Oct 2021 08:12:55 +0200 Subject: [PATCH 071/291] Handle joining a call after someone has started screen-sharing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 853ec6329a6..b59e5f517b4 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -143,6 +143,15 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.GroupCallStateChanged, newState, oldState); } + public getLocalFeeds(): CallFeed[] { + const feeds = []; + + if (this.localCallFeed) feeds.push(this.localCallFeed); + if (this.localScreenshareFeed) feeds.push(this.localScreenshareFeed); + + return feeds; + } + public async initLocalCallFeed(): Promise { if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); @@ -440,7 +449,7 @@ export class GroupCall extends EventEmitter { this.addCall(newCall); } - newCall.answerWithCallFeeds([this.localCallFeed]); + newCall.answerWithCallFeeds(this.getLocalFeeds()); }; /** @@ -533,7 +542,7 @@ export class GroupCall extends EventEmitter { { invitee: member.userId, useToDevice: true, groupCallId: this.groupCallId }, ); - newCall.placeCallWithCallFeeds([this.localCallFeed]); + newCall.placeCallWithCallFeeds(this.getLocalFeeds()); if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); From e2ed80ffa0b517a31ed0de1c15803bfd574eeccf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 3 Oct 2021 08:29:41 +0200 Subject: [PATCH 072/291] Add removeLocalFeed() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/call.ts | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b6127a11c24..92982752abb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -621,6 +621,24 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } + /** + * Removes local call feed from the call and its tracks from the peer + * connection + * @param callFeed to remove + */ + public removeLocalFeed(callFeed: CallFeed): void { + const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia + ? this.usermediaSenders + : this.screensharingSenders; + + for (const sender of senderArray) { + this.peerConn.removeTrack(sender); + } + // Empty the array + senderArray.splice(0, senderArray.length); + this.deleteFeedByStream(callFeed.stream); + } + private deleteAllFeeds(): void { for (const feed of this.feeds) { if (!feed.isLocal() || this.stopLocalMediaOnEnd) { From 8232896c859630b8ab2e9397fcd6d9f6b1a0d8af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Sun, 3 Oct 2021 08:32:25 +0200 Subject: [PATCH 073/291] Don't run screen-sharing code for each 1:1 call, share one call feed between them instead MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b59e5f517b4..97659b948d1 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -361,7 +361,8 @@ export class GroupCall extends EventEmitter { logger.log("Screensharing permissions granted. Setting screensharing enabled on all calls"); - const callFeed = new CallFeed( + this.localDesktopCapturerSourceId = desktopCapturerSourceId; + this.localScreenshareFeed = new CallFeed( stream, this.client.getUserId(), SDPStreamMetadataPurpose.Screenshare, @@ -370,10 +371,7 @@ export class GroupCall extends EventEmitter { false, false, ); - - this.localScreenshareFeed = callFeed; - this.localDesktopCapturerSourceId = desktopCapturerSourceId; - this.addScreenshareFeed(callFeed); + this.addScreenshareFeed(this.localScreenshareFeed); this.emit( GroupCallEvent.LocalScreenshareStateChanged, @@ -383,7 +381,7 @@ export class GroupCall extends EventEmitter { ); // TODO: handle errors - await Promise.all(this.calls.map(call => call.setScreensharingEnabled(true, desktopCapturerSourceId))); + await Promise.all(this.calls.map(call => call.pushLocalFeed(this.localScreenshareFeed))); logger.log("screensharing enabled on all calls"); @@ -396,7 +394,7 @@ export class GroupCall extends EventEmitter { return false; } } else { - await Promise.all(this.calls.map(call => call.setScreensharingEnabled(false, desktopCapturerSourceId))); + await Promise.all(this.calls.map(call => call.removeLocalFeed(this.localScreenshareFeed))); this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; From 27eb88f4a124dd6e4b63842a53994137f2a70d52 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 5 Oct 2021 11:38:03 -0700 Subject: [PATCH 074/291] Update GroupCall to use new CallFeed constructor --- src/webrtc/groupCall.ts | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 853ec6329a6..5aa769f8bb4 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -161,15 +161,15 @@ export class GroupCall extends EventEmitter { const userId = this.client.getUserId(); - const callFeed = new CallFeed( - stream, + const callFeed = new CallFeed({ + client: this.client, + roomId: this.room.roomId, userId, - SDPStreamMetadataPurpose.Usermedia, - this.client, - this.room.roomId, - false, - false, - ); + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: false, + }); this.activeSpeakerSamples.set(userId, Array(this.activeSpeakerSampleCount).fill( -Infinity, @@ -352,15 +352,15 @@ export class GroupCall extends EventEmitter { logger.log("Screensharing permissions granted. Setting screensharing enabled on all calls"); - const callFeed = new CallFeed( + const callFeed = new CallFeed({ + client: this.client, + roomId: this.room.roomId, + userId: this.client.getUserId(), stream, - this.client.getUserId(), - SDPStreamMetadataPurpose.Screenshare, - this.client, - this.room.roomId, - false, - false, - ); + purpose: SDPStreamMetadataPurpose.Screenshare, + audioMuted: false, + videoMuted: false, + }); this.localScreenshareFeed = callFeed; this.localDesktopCapturerSourceId = desktopCapturerSourceId; From 593f62c1c4aad5bbea16f0521b1ec41e1a4aab04 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 5 Oct 2021 16:51:28 -0700 Subject: [PATCH 075/291] Move to correct event types --- src/client.ts | 26 +++----- src/webrtc/groupCall.ts | 93 +++++++++++++++++++++-------- src/webrtc/groupCallEventHandler.ts | 61 ++++++++++++------- 3 files changed, 113 insertions(+), 67 deletions(-) diff --git a/src/client.ts b/src/client.ts index a0b2fbd1027..168827f1aa2 100644 --- a/src/client.ts +++ b/src/client.ts @@ -145,7 +145,6 @@ import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, Rule import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; import { - CALL_EVENT, GroupCall, IGroupCallDataChannelOptions, GroupCallIntent, @@ -1322,23 +1321,14 @@ export class MatrixClient extends EventEmitter { throw new Error(`Cannot find room ${roomId}`); } - const groupCall = new GroupCall(this, room, type, intent, dataChannelsEnabled, dataChannelOptions); - this.groupCallEventHandler.groupCalls.set(roomId, groupCall); - - await this.sendStateEvent( - room.roomId, - CALL_EVENT, - { - "m.intent": intent, - "m.type": type, - // TODO: Specify datachannels - dataChannelsEnabled, - dataChannelOptions, - }, - groupCall.groupCallId, - ); - - return groupCall; + return new GroupCall( + this, + room, + type, + intent, + dataChannelsEnabled, + dataChannelOptions, + ).create(); } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 511c69e9055..660512ca8e2 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -10,14 +10,13 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { createNewMatrixCall } from "./call"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; -import { RoomState } from "../models/room-state"; -export const CALL_EVENT = "org.matrix.msc3401.call"; -export const CALL_MEMBER_KEY = "org.matrix.msc3401.calls"; +export const GROUP_CALL_ROOM_EVENT = "org.matrix.msc3401.call"; +export const GROUP_CALL_MEMBER_EVENT = "org.matrix.msc3401.call.member"; export enum GroupCallIntent { Ring = "m.ring", - Prompt = "m.propmt", + Prompt = "m.prompt", Room = "m.room", } @@ -64,6 +63,16 @@ export interface IGroupCallDataChannelOptions { protocol: string; } +export interface IGroupCallRoomMemberCallState { + "m.call_id": string; + "m.foci"?: string[]; + "m.sources"?: any[]; +} + +export interface IGroupCallRoomMemberState { + "m.calls": IGroupCallRoomMemberCallState[]; +} + export enum GroupCallState { LocalCallFeedUninitialized = "local_call_feed_uninitialized", InitializingLocalCallFeed = "initializing_local_call_feed", @@ -125,16 +134,32 @@ export class GroupCall extends EventEmitter { this.groupCallId = genCallID(); const roomState = this.room.currentState; - const memberStateEvents = roomState.getStateEvents("m.room.member"); + const memberStateEvents = roomState.getStateEvents(GROUP_CALL_MEMBER_EVENT); logger.log("Processing initial members", memberStateEvents); for (const stateEvent of memberStateEvents) { - const member = this.room.getMember(stateEvent.getStateKey()); - this.onMemberStateChanged(stateEvent, roomState, member); + this.onMemberStateChanged(stateEvent); } + } + + public async create() { + this.client.groupCallEventHandler.groupCalls.set(this.room.roomId, this); - this.client.on("RoomState.members", this.onMemberStateChanged); + await this.client.sendStateEvent( + this.room.roomId, + GROUP_CALL_ROOM_EVENT, + { + "m.intent": this.intent, + "m.type": this.type, + // TODO: Specify datachannels + "dataChannelsEnabled": this.dataChannelsEnabled, + "dataChannelOptions": this.dataChannelOptions, + }, + this.groupCallId, + ); + + return this; } private setState(newState: GroupCallState): void { @@ -222,13 +247,12 @@ export class GroupCall extends EventEmitter { // Set up participants for the members currently in the room. // Other members will be picked up by the RoomState.members event. const roomState = this.room.currentState; - const memberStateEvents = roomState.getStateEvents("m.room.member"); + const memberStateEvents = roomState.getStateEvents(GROUP_CALL_MEMBER_EVENT); logger.log("Processing initial members"); for (const stateEvent of memberStateEvents) { - const member = this.room.getMember(stateEvent.getStateKey()); - this.onMemberStateChanged(stateEvent, roomState, member); + this.onMemberStateChanged(stateEvent); } this.client.on("Call.incoming", this.onIncomingCall); @@ -283,11 +307,11 @@ export class GroupCall extends EventEmitter { this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); if (emitStateEvent) { - const existingStateEvent = this.room.currentState.getStateEvents(CALL_EVENT, this.groupCallId); + const existingStateEvent = this.room.currentState.getStateEvents(GROUP_CALL_ROOM_EVENT, this.groupCallId); await this.client.sendStateEvent( this.room.roomId, - CALL_EVENT, + GROUP_CALL_ROOM_EVENT, { ...existingStateEvent.getContent(), ["m.terminated"]: GroupCallTerminationReason.CallEnded, @@ -455,34 +479,51 @@ export class GroupCall extends EventEmitter { */ private sendEnteredMemberStateEvent(): Promise { - return this.updateMemberCallsState([ - { - "m.call_id": this.groupCallId, - }, - ]); + return this.updateMemberCallState({ + "m.call_id": this.groupCallId, + // TODO "m.foci" + // TODO "m.sources" + }); } private sendLeftMemberStateEvent(): Promise { - return this.updateMemberCallsState([]); + return this.updateMemberCallState(undefined); } - private async updateMemberCallsState(state: any): Promise { + private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { const localUserId = this.client.getUserId(); - const currentStateEvent = this.room.currentState.getStateEvents("m.room.member", localUserId); + const currentStateEvent = this.room.currentState.getStateEvents(GROUP_CALL_MEMBER_EVENT, localUserId); + + const calls = currentStateEvent?.getContent()["m.calls"] || []; + + const existingCallIndex = calls.findIndex((call) => call["m.call_id"] === this.groupCallId); + + if (existingCallIndex === -1) { + calls.push(memberCallState); + } else if (memberCallState) { + calls.splice(existingCallIndex, 1, memberCallState); + } else { + calls.splice(existingCallIndex, 1); + } - return this.client.sendStateEvent(this.room.roomId, "m.room.member", { - ...currentStateEvent.getContent(), - [CALL_MEMBER_KEY]: state, + return this.client.sendStateEvent(this.room.roomId, GROUP_CALL_MEMBER_EVENT, { + "m.calls": calls, }, localUserId); } - private onMemberStateChanged = (event: MatrixEvent, state: RoomState, member: RoomMember) => { + public onMemberStateChanged = (event: MatrixEvent) => { // The member events may be received for another room, which we will ignore. if (event.getRoomId() !== this.room.roomId) { return; } + const member = this.room.getMember(event.getStateKey()); + + if (!member) { + return; + } + // Don't process your own member. const localUserId = this.client.getUserId(); @@ -490,7 +531,7 @@ export class GroupCall extends EventEmitter { return; } - const callsState = event.getContent()[CALL_MEMBER_KEY]; + const callsState = event.getContent()["m.calls"]; if (!callsState || !Array.isArray(callsState) || callsState.length === 0) { logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 657727203ad..c932a4ccd47 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -16,7 +16,14 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { MatrixClient } from '../client'; -import { CALL_EVENT, GroupCall, GroupCallIntent, GroupCallType, IGroupCallDataChannelOptions } from "./groupCall"; +import { + GROUP_CALL_ROOM_EVENT, + GROUP_CALL_MEMBER_EVENT, + GroupCall, + GroupCallIntent, + GroupCallType, + IGroupCallDataChannelOptions, +} from "./groupCall"; import { Room } from "../models/room"; import { RoomState } from "../models/room-state"; import { logger } from '../logger'; @@ -46,7 +53,7 @@ export class GroupCallEventHandler { } private createGroupCallForRoom(room: Room): GroupCall | undefined { - const callEvents = room.currentState.getStateEvents(CALL_EVENT); + const callEvents = room.currentState.getStateEvents(GROUP_CALL_ROOM_EVENT); const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); for (const callEvent of sortedCallEvents) { @@ -118,29 +125,37 @@ export class GroupCallEventHandler { }; private onRoomStateChanged = (event: MatrixEvent, state: RoomState): void => { - if (event.getType() !== CALL_EVENT) { - return; - } - - const groupCallId = event.getStateKey(); - const content = event.getContent(); - - const currentGroupCall = this.groupCalls.get(state.roomId); + const eventType = event.getType(); + + if (eventType === GROUP_CALL_ROOM_EVENT) { + const groupCallId = event.getStateKey(); + const content = event.getContent(); + + const currentGroupCall = this.groupCalls.get(state.roomId); + + if (!currentGroupCall && !content["m.terminated"]) { + this.createGroupCallFromRoomStateEvent(event); + } else if (currentGroupCall && currentGroupCall.groupCallId === groupCallId) { + if (content["m.terminated"]) { + currentGroupCall.terminate(false); + } else if (content["m.type"] !== currentGroupCall.type) { + // TODO: Handle the callType changing when the room state changes + logger.warn(`The group call type changed for room: ${ + state.roomId}. Changing the group call type is currently unsupported.`); + } + } else if (currentGroupCall && currentGroupCall.groupCallId !== groupCallId) { + // TODO: Handle new group calls and multiple group calls + logger.warn(`Multiple group calls detected for room: ${ + state.roomId}. Multiple group calls are currently unsupported.`); + } + } else if (eventType === GROUP_CALL_MEMBER_EVENT) { + const groupCall = this.groupCalls.get(state.roomId); - if (!currentGroupCall && !content["m.terminated"]) { - this.createGroupCallFromRoomStateEvent(event); - } else if (currentGroupCall && currentGroupCall.groupCallId === groupCallId) { - if (content["m.terminated"]) { - currentGroupCall.terminate(false); - } else if (content["m.type"] !== currentGroupCall.type) { - // TODO: Handle the callType changing when the room state changes - logger.warn(`The group call type changed for room: ${ - state.roomId}. Changing the group call type is currently unsupported.`); + if (!groupCall) { + return; } - } else if (currentGroupCall && currentGroupCall.groupCallId !== groupCallId) { - // TODO: Handle new group calls and multiple group calls - logger.warn(`Multiple group calls detected for room: ${ - state.roomId}. Multiple group calls are currently unsupported.`); + + groupCall.onMemberStateChanged(event); } }; } From 7ef38ed1b2f0ea030c40eb0c9e6df1dcdfc54a76 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 6 Oct 2021 16:20:07 -0700 Subject: [PATCH 076/291] Fix speaking threshold --- src/webrtc/callFeed.ts | 2 +- src/webrtc/groupCall.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 0e126dfc841..109b319bb0a 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -20,7 +20,7 @@ import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; const POLLING_INTERVAL = 250; // ms -const SPEAKING_THRESHOLD = -60; // dB +export const SPEAKING_THRESHOLD = -60; // dB export interface ICallFeedOpts { client: MatrixClient; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 660512ca8e2..3f999c28a51 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,5 +1,5 @@ import EventEmitter from "events"; -import { CallFeed, CallFeedEvent } from "./callFeed"; +import { CallFeed, CallFeedEvent, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient } from "../client"; import { CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; @@ -101,7 +101,7 @@ export class GroupCall extends EventEmitter { // Config public activeSpeakerSampleCount = 8; public activeSpeakerInterval = 1000; - public speakingThreshold = -80; + public speakingThreshold = SPEAKING_THRESHOLD; public participantTimeout = 1000 * 15; public state = GroupCallState.LocalCallFeedUninitialized; From 8dc608d91741fac6f1b33a76c9a3acd2ee4183c9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 7 Oct 2021 13:47:50 -0700 Subject: [PATCH 077/291] Fix connecting to a call without a webcam --- src/webrtc/call.ts | 20 ++++++++++++++------ src/webrtc/groupCall.ts | 16 ++++++++++++---- src/webrtc/mediaHandler.ts | 24 +++++++++++++++++++++--- 3 files changed, 47 insertions(+), 13 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index fd6b27e7b6a..042b8e6dffe 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -598,8 +598,8 @@ export class MatrixCall extends EventEmitter { this.pushLocalFeed(new CallFeed({ client: this.client, roomId: this.roomId, - audioMuted: false, - videoMuted: false, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, userId, stream, purpose, @@ -832,8 +832,8 @@ export class MatrixCall extends EventEmitter { userId: this.client.getUserId(), stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, }); this.answerWithCallFeeds([callFeed], true); } catch (e) { @@ -1077,6 +1077,10 @@ export class MatrixCall extends EventEmitter { * @returns the new mute state */ public async setLocalVideoMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return this.isLocalVideoMuted(); + } + if (!this.hasLocalUserMediaVideoTrack && !muted) { await this.upgradeCall(false, true); return this.isLocalVideoMuted(); @@ -1105,6 +1109,10 @@ export class MatrixCall extends EventEmitter { * @returns the new mute state */ public async setMicrophoneMuted(muted: boolean): Promise { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return this.isMicrophoneMuted(); + } + if (!this.hasLocalUserMediaAudioTrack && !muted) { await this.upgradeCall(true, false); return this.isMicrophoneMuted(); @@ -2083,8 +2091,8 @@ export class MatrixCall extends EventEmitter { userId: this.client.getUserId(), stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, }); await this.placeCallWithCallFeeds([callFeed], true); } catch (e) { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3f999c28a51..ae57f719f60 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -201,8 +201,8 @@ export class GroupCall extends EventEmitter { userId, stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, + audioMuted: stream.getAudioTracks().length === 0, + videoMuted: stream.getVideoTracks().length === 0, }); this.activeSpeakerSamples.set(userId, Array(this.activeSpeakerSampleCount).fill( @@ -344,7 +344,11 @@ export class GroupCall extends EventEmitter { return true; } - public setMicrophoneMuted(muted) { + public async setMicrophoneMuted(muted) { + if (!await this.client.getMediaHandler().hasAudioDevice()) { + return false; + } + if (this.localCallFeed) { this.localCallFeed.setAudioMuted(muted); setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); @@ -357,7 +361,11 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); } - public setLocalVideoMuted(muted) { + public async setLocalVideoMuted(muted) { + if (!await this.client.getMediaHandler().hasVideoDevice()) { + return false; + } + if (this.localCallFeed) { this.localCallFeed.setVideoMuted(muted); setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 3f3dcaad62a..d975329e0d4 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -43,16 +43,34 @@ export class MediaHandler { this.videoInput = deviceId; } + public async hasAudioDevice() { + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter(device => device.kind === "audioinput").length > 0; + } + + public async hasVideoDevice() { + const devices = await navigator.mediaDevices.enumerateDevices(); + return devices.filter(device => device.kind === "videoinput").length > 0; + } + /** * @returns {MediaStream} based on passed parameters */ public async getUserMediaStream(audio: boolean, video: boolean): Promise { + const devices = await navigator.mediaDevices.enumerateDevices(); + + const audioDevices = devices.filter(device => device.kind === "audioinput"); + const videoDevices = devices.filter(device => device.kind === "videoinput"); + + const shouldRequestAudio = audio && audioDevices.length > 0; + const shouldRequestVideo = video && videoDevices.length > 0; + let stream: MediaStream; // Find a stream with matching tracks const matchingStream = this.userMediaStreams.find((stream) => { - if (audio !== (stream.getAudioTracks().length > 0)) return false; - if (video !== (stream.getVideoTracks().length > 0)) return false; + if (shouldRequestAudio !== (stream.getAudioTracks().length > 0)) return false; + if (shouldRequestVideo !== (stream.getVideoTracks().length > 0)) return false; return true; }); @@ -60,7 +78,7 @@ export class MediaHandler { logger.log("Cloning user media stream", matchingStream.id); stream = matchingStream.clone(); } else { - const constraints = this.getUserMediaContraints(audio, video); + const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); logger.log("Getting user media with constraints", constraints); stream = await navigator.mediaDevices.getUserMedia(constraints); } From 4b6b1599a2c6d36b331b61c631c291fb5a203436 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 12 Oct 2021 14:11:57 -0700 Subject: [PATCH 078/291] Change media devices mid-call --- src/client.ts | 2 +- src/webrtc/call.ts | 17 +++++++++++++++++ src/webrtc/mediaHandler.ts | 19 +++++++++++++++---- 3 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/client.ts b/src/client.ts index b1a2bad2305..52a81cc9594 100644 --- a/src/client.ts +++ b/src/client.ts @@ -743,7 +743,7 @@ export class MatrixClient extends EventEmitter { protected checkTurnServersIntervalID: number; protected exportedOlmDeviceToImport: IOlmDevice; protected txnCtr = 0; - protected mediaHandler = new MediaHandler(); + protected mediaHandler = new MediaHandler(this); constructor(opts: IMatrixClientCreateOpts) { super(); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7afa8f08ae6..69fe6dc6715 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -998,6 +998,23 @@ export class MatrixCall extends EventEmitter { } } + /** + * Request a new local usermedia stream with the current device id. + */ + public async updateLocalUsermediaStream() { + const oldStream = this.localUsermediaStream; + + const stream = await this.client.getMediaHandler().getUserMediaStream( + this.hasLocalUserMediaAudioTrack, + this.hasLocalUserMediaVideoTrack, + true, + ); + + this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia, true); + + this.client.getMediaHandler().stopUserMediaStream(oldStream); + } + /** * Set whether our outbound video should be muted or not. * @param {boolean} muted True to mute the outbound video. diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 4a20823bd48..8cebff8ea65 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -17,6 +17,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { MatrixClient } from "../client"; import { logger } from "../logger"; export class MediaHandler { @@ -25,13 +26,19 @@ export class MediaHandler { private userMediaStreams: MediaStream[] = []; private screensharingStreams: MediaStream[] = []; + constructor(private client: MatrixClient) {} + /** * Set an audio input device to use for MatrixCalls * @param {string} deviceId the identifier for the device * undefined treated as unset */ - public setAudioInput(deviceId: string): void { + public async setAudioInput(deviceId: string): Promise { this.audioInput = deviceId; + + await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { + return call.updateLocalUsermediaStream(); + })); } /** @@ -39,8 +46,12 @@ export class MediaHandler { * @param {string} deviceId the identifier for the device * undefined treated as unset */ - public setVideoInput(deviceId: string): void { + public async setVideoInput(deviceId: string): Promise { this.videoInput = deviceId; + + await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { + return call.updateLocalUsermediaStream(); + })); } public async hasAudioDevice(): Promise { @@ -56,7 +67,7 @@ export class MediaHandler { /** * @returns {MediaStream} based on passed parameters */ - public async getUserMediaStream(audio: boolean, video: boolean): Promise { + public async getUserMediaStream(audio: boolean, video: boolean, forceNewStream = false): Promise { const shouldRequestAudio = audio && await this.hasAudioDevice(); const shouldRequestVideo = video && await this.hasVideoDevice(); @@ -69,7 +80,7 @@ export class MediaHandler { return true; }); - if (matchingStream) { + if (matchingStream && !forceNewStream) { logger.log("Cloning user media stream", matchingStream.id); stream = matchingStream.clone(); } else { From a0f6eea363cf1d62238746dbd264cc2301c3224e Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 12 Oct 2021 14:48:59 -0700 Subject: [PATCH 079/291] Add support for replacing existing sender tracks --- src/webrtc/call.ts | 41 ++++++++++++++++++++++++++++++++++++++++- 1 file changed, 40 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 69fe6dc6715..50ed1182fcc 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1010,7 +1010,46 @@ export class MatrixCall extends EventEmitter { true, ); - this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia, true); + const callFeed = this.localUsermediaFeed; + + callFeed.setNewStream(stream); + + const newSenders = []; + + for (const track of stream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; this.client.getMediaHandler().stopUserMediaStream(oldStream); } From ec37eb8b6f7802991a4310a26dcdecad43dee3fd Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 12 Oct 2021 15:20:14 -0700 Subject: [PATCH 080/291] Add support for switching media devices --- src/webrtc/groupCall.ts | 10 ++++++++++ src/webrtc/mediaHandler.ts | 8 ++++++++ 2 files changed, 18 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index ae57f719f60..536515e6484 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -216,6 +216,16 @@ export class GroupCall extends EventEmitter { return callFeed; } + public async updateLocalUsermediaStream() { + if (this.localCallFeed) { + const mediaHandler = this.client.getMediaHandler(); + const oldStream = this.localCallFeed.stream; + const stream = await mediaHandler.getUserMediaStream(true, this.type === GroupCallType.Video, true); + this.localCallFeed.setNewStream(stream); + mediaHandler.stopUserMediaStream(oldStream); + } + } + public async enter() { if (!(this.state === GroupCallState.LocalCallFeedUninitialized || this.state === GroupCallState.LocalCallFeedInitialized)) { diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index e0e69ffd71c..40dcfcbb9dc 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -39,6 +39,10 @@ export class MediaHandler { await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { return call.updateLocalUsermediaStream(); })); + + await Promise.all(Array.from(this.client.groupCallEventHandler.groupCalls.values()).map((groupCall) => { + return groupCall.updateLocalUsermediaStream(); + })); } /** @@ -52,6 +56,10 @@ export class MediaHandler { await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { return call.updateLocalUsermediaStream(); })); + + await Promise.all(Array.from(this.client.groupCallEventHandler.groupCalls.values()).map((groupCall) => { + return groupCall.updateLocalUsermediaStream(); + })); } public async hasAudioDevice(): Promise { From 2d231c0ae25332be50dbac880e323540ac5c557e Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 13 Oct 2021 12:05:14 -0700 Subject: [PATCH 081/291] Fix how streams are stopped --- src/webrtc/call.ts | 4 ---- src/webrtc/groupCall.ts | 2 -- src/webrtc/mediaHandler.ts | 12 ++++++++++++ 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3824cc4e191..b515482c3d8 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1086,8 +1086,6 @@ export class MatrixCall extends EventEmitter { * Request a new local usermedia stream with the current device id. */ public async updateLocalUsermediaStream() { - const oldStream = this.localUsermediaStream; - const stream = await this.client.getMediaHandler().getUserMediaStream( this.hasLocalUserMediaAudioTrack, this.hasLocalUserMediaVideoTrack, @@ -1134,8 +1132,6 @@ export class MatrixCall extends EventEmitter { } this.usermediaSenders = newSenders; - - this.client.getMediaHandler().stopUserMediaStream(oldStream); } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 536515e6484..e79e10d192c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -219,10 +219,8 @@ export class GroupCall extends EventEmitter { public async updateLocalUsermediaStream() { if (this.localCallFeed) { const mediaHandler = this.client.getMediaHandler(); - const oldStream = this.localCallFeed.stream; const stream = await mediaHandler.getUserMediaStream(true, this.type === GroupCallType.Video, true); this.localCallFeed.setNewStream(stream); - mediaHandler.stopUserMediaStream(oldStream); } } diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 40dcfcbb9dc..38308270739 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -36,6 +36,12 @@ export class MediaHandler { public async setAudioInput(deviceId: string): Promise { this.audioInput = deviceId; + for (const stream of this.userMediaStreams) { + for (const track of stream.getTracks()) { + track.stop(); + } + } + await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { return call.updateLocalUsermediaStream(); })); @@ -53,6 +59,12 @@ export class MediaHandler { public async setVideoInput(deviceId: string): Promise { this.videoInput = deviceId; + for (const stream of this.userMediaStreams) { + for (const track of stream.getTracks()) { + track.stop(); + } + } + await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { return call.updateLocalUsermediaStream(); })); From 411b5f111c26ae2d7dd4738036bb7abf1bd7899d Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 13 Oct 2021 13:42:40 -0700 Subject: [PATCH 082/291] Fix talking indicator --- src/webrtc/callFeed.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 109b319bb0a..429f04d86d6 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -78,6 +78,10 @@ export class CallFeed extends EventEmitter { } private updateStream(oldStream: MediaStream, newStream: MediaStream): void { + if (newStream === oldStream) { + return; + } + if (oldStream) { oldStream.removeEventListener("addtrack", this.onAddTrack); this.measureVolumeActivity(false); From 5c8e7f2be0c31910a8dc47e595000f1d81493959 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 14 Oct 2021 13:34:20 -0700 Subject: [PATCH 083/291] Improve speaking detection using history --- src/webrtc/callFeed.ts | 53 +++++++++++++++++---------- src/webrtc/groupCall.ts | 80 +++++++---------------------------------- 2 files changed, 47 insertions(+), 86 deletions(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 429f04d86d6..6a9df482831 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -19,8 +19,9 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; -const POLLING_INTERVAL = 250; // ms +const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB +const SPEAKING_SAMPLE_COUNT = 8; export interface ICallFeedOpts { client: MatrixClient; @@ -55,6 +56,7 @@ export class CallFeed extends EventEmitter { private speakingThreshold = SPEAKING_THRESHOLD; private speaking = false; private volumeLooperTimeout: number; + public speakingVolumeSamples: number[]; constructor(opts: ICallFeedOpts) { super(); @@ -114,6 +116,7 @@ export class CallFeed extends EventEmitter { mediaStreamAudioSourceNode.connect(this.analyser); this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); } private onAddTrack = (): void => { @@ -175,6 +178,7 @@ export class CallFeed extends EventEmitter { */ public setAudioMuted(muted: boolean): void { this.audioMuted = muted; + this.speakingVolumeSamples.fill(-Infinity); this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } @@ -199,6 +203,7 @@ export class CallFeed extends EventEmitter { this.volumeLooper(); } else { this.measuringVolumeActivity = false; + this.speakingVolumeSamples.fill(-Infinity); this.emit(CallFeedEvent.VolumeChanged, -Infinity); } } @@ -207,31 +212,43 @@ export class CallFeed extends EventEmitter { this.speakingThreshold = threshold; } - private volumeLooper(): void { + private volumeLooper = () => { if (!this.analyser) return; - this.volumeLooperTimeout = setTimeout(() => { - if (!this.measuringVolumeActivity) return; + if (!this.measuringVolumeActivity) return; - this.analyser.getFloatFrequencyData(this.frequencyBinCount); + this.analyser.getFloatFrequencyData(this.frequencyBinCount); - let maxVolume = -Infinity; - for (let i = 0; i < this.frequencyBinCount.length; i++) { - if (this.frequencyBinCount[i] > maxVolume) { - maxVolume = this.frequencyBinCount[i]; - } + let maxVolume = -Infinity; + for (let i = 0; i < this.frequencyBinCount.length; i++) { + if (this.frequencyBinCount[i] > maxVolume) { + maxVolume = this.frequencyBinCount[i]; } + } + + this.speakingVolumeSamples.shift(); + this.speakingVolumeSamples.push(maxVolume); + + this.emit(CallFeedEvent.VolumeChanged, maxVolume); - this.emit(CallFeedEvent.VolumeChanged, maxVolume); - const newSpeaking = maxVolume > this.speakingThreshold; - if (this.speaking !== newSpeaking) { - this.speaking = newSpeaking; - this.emit(CallFeedEvent.Speaking, this.speaking); + let newSpeaking = false; + + for (let i = 0; i < this.speakingVolumeSamples.length; i++) { + const volume = this.speakingVolumeSamples[i]; + + if (volume > this.speakingThreshold) { + newSpeaking = true; + break; } + } - this.volumeLooper(); - }, POLLING_INTERVAL); - } + if (this.speaking !== newSpeaking) { + this.speaking = newSpeaking; + this.emit(CallFeedEvent.Speaking, this.speaking); + } + + this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); + }; public dispose(): void { clearTimeout(this.volumeLooperTimeout); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e79e10d192c..baa3ab63094 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,5 +1,5 @@ import EventEmitter from "events"; -import { CallFeed, CallFeedEvent, SPEAKING_THRESHOLD } from "./callFeed"; +import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient } from "../client"; import { CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; @@ -82,11 +82,6 @@ export enum GroupCallState { Ended = "ended", } -interface IUserMediaFeedHandlers { - onCallFeedVolumeChanged: (maxVolume: number) => void; - onCallFeedMuteStateChanged: (audioMuted: boolean) => void; -} - interface ICallHandlers { onCallFeedsChanged: (feeds: CallFeed[]) => void; onCallStateChanged: (state: CallState, oldState: CallState) => void; @@ -99,9 +94,7 @@ function getCallUserId(call: MatrixCall): string | null { export class GroupCall extends EventEmitter { // Config - public activeSpeakerSampleCount = 8; public activeSpeakerInterval = 1000; - public speakingThreshold = SPEAKING_THRESHOLD; public participantTimeout = 1000 * 15; public state = GroupCallState.LocalCallFeedUninitialized; @@ -115,9 +108,7 @@ export class GroupCall extends EventEmitter { public screenshareFeeds: CallFeed[] = []; public groupCallId: string; - private userMediaFeedHandlers: Map = new Map(); private callHandlers: Map = new Map(); - private activeSpeakerSamples: Map = new Map(); private activeSpeakerLoopTimeout?: number; private reEmitter: ReEmitter; @@ -205,9 +196,6 @@ export class GroupCall extends EventEmitter { videoMuted: stream.getVideoTracks().length === 0, }); - this.activeSpeakerSamples.set(userId, Array(this.activeSpeakerSampleCount).fill( - -Infinity, - )); this.localCallFeed = callFeed; this.addUserMediaFeed(callFeed); @@ -680,9 +668,6 @@ export class GroupCall extends EventEmitter { call.setScreensharingEnabled(true, this.localDesktopCapturerSourceId); } - this.activeSpeakerSamples.set(opponentMemberId, Array(this.activeSpeakerSampleCount).fill( - -Infinity, - )); this.reEmitter.reEmit(call, Object.values(CallEvent)); } @@ -720,8 +705,6 @@ export class GroupCall extends EventEmitter { if (screenshareFeed) { this.removeScreenshareFeed(screenshareFeed); } - - this.activeSpeakerSamples.delete(opponentMemberId); } private onCallFeedsChanged = (call: MatrixCall) => { @@ -798,7 +781,7 @@ export class GroupCall extends EventEmitter { private addUserMediaFeed(callFeed: CallFeed) { this.userMediaFeeds.push(callFeed); - this.initUserMediaFeed(callFeed); + callFeed.measureVolumeActivity(true); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } @@ -811,8 +794,8 @@ export class GroupCall extends EventEmitter { this.userMediaFeeds.splice(feedIndex, 1, replacementFeed); - this.disposeUserMediaFeed(existingFeed); - this.initUserMediaFeed(replacementFeed); + existingFeed.dispose(); + replacementFeed.measureVolumeActivity(true); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); } @@ -825,7 +808,7 @@ export class GroupCall extends EventEmitter { this.userMediaFeeds.splice(feedIndex, 1); - this.disposeUserMediaFeed(callFeed); + callFeed.dispose(); this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); if ( @@ -837,66 +820,27 @@ export class GroupCall extends EventEmitter { } } - private initUserMediaFeed(callFeed: CallFeed) { - callFeed.setSpeakingThreshold(this.speakingThreshold); - callFeed.measureVolumeActivity(true); - - const onCallFeedVolumeChanged = (maxVolume: number) => this.onCallFeedVolumeChanged(callFeed, maxVolume); - const onCallFeedMuteStateChanged = - (audioMuted: boolean) => this.onCallFeedMuteStateChanged(callFeed, audioMuted); - - this.userMediaFeedHandlers.set(callFeed.userId, { - onCallFeedVolumeChanged, - onCallFeedMuteStateChanged, - }); - - callFeed.on(CallFeedEvent.VolumeChanged, onCallFeedVolumeChanged); - callFeed.on(CallFeedEvent.MuteStateChanged, onCallFeedMuteStateChanged); - } - - private disposeUserMediaFeed(callFeed: CallFeed) { - const { onCallFeedVolumeChanged, onCallFeedMuteStateChanged } = this.userMediaFeedHandlers.get(callFeed.userId); - callFeed.removeListener(CallFeedEvent.VolumeChanged, onCallFeedVolumeChanged); - callFeed.removeListener(CallFeedEvent.MuteStateChanged, onCallFeedMuteStateChanged); - this.userMediaFeedHandlers.delete(callFeed.userId); - callFeed.dispose(); - } - - private onCallFeedVolumeChanged = (callFeed: CallFeed, maxVolume: number) => { - const activeSpeakerSamples = this.activeSpeakerSamples.get(callFeed.userId); - activeSpeakerSamples.shift(); - activeSpeakerSamples.push(maxVolume); - }; - - private onCallFeedMuteStateChanged = (callFeed: CallFeed, audioMuted: boolean) => { - if (audioMuted) { - this.activeSpeakerSamples.get(callFeed.userId).fill( - -Infinity, - ); - } - }; - private onActiveSpeakerLoop = () => { let topAvg: number; let nextActiveSpeaker: string; - for (const [userId, samples] of this.activeSpeakerSamples) { + for (const callFeed of this.userMediaFeeds) { let total = 0; - for (let i = 0; i < samples.length; i++) { - const volume = samples[i]; - total += Math.max(volume, this.speakingThreshold); + for (let i = 0; i < callFeed.speakingVolumeSamples.length; i++) { + const volume = callFeed.speakingVolumeSamples[i]; + total += Math.max(volume, SPEAKING_THRESHOLD); } - const avg = total / this.activeSpeakerSampleCount; + const avg = total / callFeed.speakingVolumeSamples.length; if (!topAvg || avg > topAvg) { topAvg = avg; - nextActiveSpeaker = userId; + nextActiveSpeaker = callFeed.userId; } } - if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg > this.speakingThreshold) { + if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg > SPEAKING_THRESHOLD) { this.activeSpeaker = nextActiveSpeaker; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } From 015d0f91358a8d0d70a6cb078cd80ee882d43f3d Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 14 Oct 2021 17:14:12 -0700 Subject: [PATCH 084/291] Don't set local user as active speaker --- src/webrtc/groupCall.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index baa3ab63094..68d1118263b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -226,7 +226,7 @@ export class GroupCall extends EventEmitter { this.sendEnteredMemberStateEvent(); - this.activeSpeaker = this.client.getUserId(); + this.activeSpeaker = null; this.setState(GroupCallState.Entered); @@ -825,6 +825,10 @@ export class GroupCall extends EventEmitter { let nextActiveSpeaker: string; for (const callFeed of this.userMediaFeeds) { + if (callFeed.userId === this.client.getUserId() && this.userMediaFeeds.length > 1) { + continue; + } + let total = 0; for (let i = 0; i < callFeed.speakingVolumeSamples.length; i++) { From debeb66d6f0095fe8c070fe4eae2d988204d9aac Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 14 Oct 2021 17:24:05 -0700 Subject: [PATCH 085/291] Initialize speakingVolumeSamples for screenshare feeds --- src/webrtc/callFeed.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 6a9df482831..1636540c668 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -67,6 +67,7 @@ export class CallFeed extends EventEmitter { this.purpose = opts.purpose; this.audioMuted = opts.audioMuted; this.videoMuted = opts.videoMuted; + this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); this.updateStream(null, opts.stream); @@ -116,7 +117,6 @@ export class CallFeed extends EventEmitter { mediaStreamAudioSourceNode.connect(this.analyser); this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount); - this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); } private onAddTrack = (): void => { From 843973c4daa1f2c99ff98ee2f45278de414be96e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 15 Oct 2021 17:19:22 +0200 Subject: [PATCH 086/291] Remove left-over from old screen-sharing code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 68d1118263b..0f99e949164 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -664,10 +664,6 @@ export class GroupCall extends EventEmitter { call.on(CallEvent.State, onCallStateChanged); call.on(CallEvent.Hangup, onCallHangup); - if (this.isScreensharing()) { - call.setScreensharingEnabled(true, this.localDesktopCapturerSourceId); - } - this.reEmitter.reEmit(call, Object.values(CallEvent)); } From 50e6a8f6b1f43b5ff7e3ac1f8cc191b0d55d4b1b Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 18 Oct 2021 11:49:04 -0700 Subject: [PATCH 087/291] Add session_id check to group calls --- src/webrtc/groupCall.ts | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0f99e949164..98a0b536b1a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -65,6 +65,7 @@ export interface IGroupCallDataChannelOptions { export interface IGroupCallRoomMemberCallState { "m.call_id": string; + "m.session_id": string; "m.foci"?: string[]; "m.sources"?: any[]; } @@ -108,6 +109,8 @@ export class GroupCall extends EventEmitter { public screenshareFeeds: CallFeed[] = []; public groupCallId: string; + private sessionId: string; + private sessionIds: Map = new Map(); private callHandlers: Map = new Map(); private activeSpeakerLoopTimeout?: number; private reEmitter: ReEmitter; @@ -123,6 +126,7 @@ export class GroupCall extends EventEmitter { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = genCallID(); + this.sessionId = genCallID(); const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(GROUP_CALL_MEMBER_EVENT); @@ -485,6 +489,7 @@ export class GroupCall extends EventEmitter { private sendEnteredMemberStateEvent(): Promise { return this.updateMemberCallState({ "m.call_id": this.groupCallId, + "m.session_id": this.sessionId, // TODO "m.foci" // TODO "m.sources" }); @@ -524,6 +529,8 @@ export class GroupCall extends EventEmitter { const member = this.room.getMember(event.getStateKey()); + logger.log("Processing member state", member); + if (!member) { return; } @@ -560,15 +567,15 @@ export class GroupCall extends EventEmitter { return; } - this.addParticipant(member); + const sessionId = callState["m.session_id"]; - if (this.state !== GroupCallState.Entered) { + if (!sessionId || sessionId === this.sessionIds.get(member.userId)) { return; } - const existingCall = this.getCallByUserId(member.userId); + this.addParticipant(member, sessionId); - if (existingCall) { + if (this.state !== GroupCallState.Entered) { return; } @@ -579,6 +586,8 @@ export class GroupCall extends EventEmitter { return; } + const existingCall = this.getCallByUserId(member.userId); + const newCall = createNewMatrixCall( this.client, this.room.roomId, @@ -591,7 +600,6 @@ export class GroupCall extends EventEmitter { newCall.createDataChannel("datachannel", this.dataChannelOptions); } - // TODO: This existingCall code path is never reached, do we still need it? if (existingCall) { this.replaceCall(existingCall, newCall); } else { @@ -897,12 +905,13 @@ export class GroupCall extends EventEmitter { * Participant Management */ - private addParticipant(member: RoomMember) { + private addParticipant(member: RoomMember, sessionId: string) { if (this.participants.find((m) => m.userId === member.userId)) { return; } this.participants.push(member); + this.sessionIds.set(member.userId, sessionId); this.emit(GroupCallEvent.ParticipantsChanged, this.participants); this.client.emit("GroupCall.participants", this.participants, this); @@ -916,6 +925,7 @@ export class GroupCall extends EventEmitter { } this.participants.splice(index, 1); + this.sessionIds.delete(member.userId); this.emit(GroupCallEvent.ParticipantsChanged, this.participants); this.client.emit("GroupCall.participants", this.participants, this); From 213f1134b6d5634a709f65b23fa9ff83f346a6dc Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 18 Oct 2021 11:49:15 -0700 Subject: [PATCH 088/291] Reduce logging for group calls --- src/webrtc/callEventHandler.ts | 7 ------- src/webrtc/groupCallEventHandler.ts | 2 -- 2 files changed, 9 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index c1bbaafab15..2454c1dafd3 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -131,7 +131,6 @@ export class CallEventHandler { }; private onToDeviceEvent = (event: MatrixEvent): void => { - logger.log("onToDeviceEvent", event); if (!this.eventIsACall(event)) return; this.toDeviceCallEventBuffer.push(event); @@ -140,8 +139,6 @@ export class CallEventHandler { private async evaluateToDeviceEventBuffer(): Promise { if (this.client.getSyncState() !== SyncState.Syncing) return; - logger.log("processing to device events"); - for (const event of this.toDeviceCallEventBuffer) { try { await this.handleCallEvent(event, true); @@ -150,8 +147,6 @@ export class CallEventHandler { } } - logger.log("processing to device events finished"); - this.toDeviceCallEventBuffer = []; } @@ -176,8 +171,6 @@ export class CallEventHandler { let call = content.call_id ? this.calls.get(content.call_id) : undefined; //console.info("RECV %s content=%s", type, JSON.stringify(content)); - logger.log("handleCallEvent", callRoomId, groupCallId, type, event); - if (!callRoomId) return; if (type === EventType.CallInvite) { diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index c932a4ccd47..196d3c0fdde 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -71,8 +71,6 @@ export class GroupCallEventHandler { const roomId = event.getRoomId(); const content = event.getContent(); - logger.log("createGroupCallFromRoomStateEvent", roomId); - const room = this.client.getRoom(roomId); if (!room) { From 0148ad0766fd74ce83f8c11465ca33ea92d1762f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 19 Oct 2021 19:25:47 +0200 Subject: [PATCH 089/291] Group call improvements (#1985) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add group call events to EventType Signed-off-by: Šimon Brandner * Use EventType instead of a const Signed-off-by: Šimon Brandner * Make logging around sending group call member state event a bit better Signed-off-by: Šimon Brandner * Fix m.calls elements being null Signed-off-by: Šimon Brandner --- src/@types/event.ts | 4 +++ src/webrtc/groupCall.ts | 38 +++++++++++++++++------------ src/webrtc/groupCallEventHandler.ts | 9 +++---- 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/@types/event.ts b/src/@types/event.ts index f23d7e76727..56c0025089b 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -87,6 +87,10 @@ export enum EventType { RoomKeyRequest = "m.room_key_request", ForwardedRoomKey = "m.forwarded_room_key", Dummy = "m.dummy", + + // Group call events + GroupCallPrefix = "org.matrix.msc3401.call", + GroupCallMemberPrefix = "org.matrix.msc3401.call.member", } export enum RelationType { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 98a0b536b1a..b3355480058 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -10,9 +10,7 @@ import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { createNewMatrixCall } from "./call"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; - -export const GROUP_CALL_ROOM_EVENT = "org.matrix.msc3401.call"; -export const GROUP_CALL_MEMBER_EVENT = "org.matrix.msc3401.call.member"; +import { EventType } from "../@types/event"; export enum GroupCallIntent { Ring = "m.ring", @@ -129,7 +127,7 @@ export class GroupCall extends EventEmitter { this.sessionId = genCallID(); const roomState = this.room.currentState; - const memberStateEvents = roomState.getStateEvents(GROUP_CALL_MEMBER_EVENT); + const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); logger.log("Processing initial members", memberStateEvents); @@ -143,7 +141,7 @@ export class GroupCall extends EventEmitter { await this.client.sendStateEvent( this.room.roomId, - GROUP_CALL_ROOM_EVENT, + EventType.GroupCallPrefix, { "m.intent": this.intent, "m.type": this.type, @@ -226,8 +224,6 @@ export class GroupCall extends EventEmitter { await this.initLocalCallFeed(); } - logger.log(`Sending member state event with current call.`); - this.sendEnteredMemberStateEvent(); this.activeSpeaker = null; @@ -247,7 +243,7 @@ export class GroupCall extends EventEmitter { // Set up participants for the members currently in the room. // Other members will be picked up by the RoomState.members event. const roomState = this.room.currentState; - const memberStateEvents = roomState.getStateEvents(GROUP_CALL_MEMBER_EVENT); + const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); logger.log("Processing initial members"); @@ -307,11 +303,13 @@ export class GroupCall extends EventEmitter { this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); if (emitStateEvent) { - const existingStateEvent = this.room.currentState.getStateEvents(GROUP_CALL_ROOM_EVENT, this.groupCallId); + const existingStateEvent = this.room.currentState.getStateEvents( + EventType.GroupCallPrefix, this.groupCallId, + ); await this.client.sendStateEvent( this.room.roomId, - GROUP_CALL_ROOM_EVENT, + EventType.GroupCallPrefix, { ...existingStateEvent.getContent(), ["m.terminated"]: GroupCallTerminationReason.CallEnded, @@ -502,11 +500,17 @@ export class GroupCall extends EventEmitter { private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { const localUserId = this.client.getUserId(); - const currentStateEvent = this.room.currentState.getStateEvents(GROUP_CALL_MEMBER_EVENT, localUserId); + const currentStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, localUserId); + const memberStateEvent = currentStateEvent?.getContent(); - const calls = currentStateEvent?.getContent()["m.calls"] || []; - - const existingCallIndex = calls.findIndex((call) => call["m.call_id"] === this.groupCallId); + let calls: IGroupCallRoomMemberCallState[] = []; + let existingCallIndex: number; + if (memberCallState) { + calls = memberStateEvent["m.calls"] || []; + existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); + } else { + existingCallIndex = 0; + } if (existingCallIndex === -1) { calls.push(memberCallState); @@ -516,9 +520,11 @@ export class GroupCall extends EventEmitter { calls.splice(existingCallIndex, 1); } - return this.client.sendStateEvent(this.room.roomId, GROUP_CALL_MEMBER_EVENT, { + const content = { "m.calls": calls, - }, localUserId); + }; + logger.log("Sending group call member state event", content); + return this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId); } public onMemberStateChanged = (event: MatrixEvent) => { diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 196d3c0fdde..fd76299b0ea 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -17,8 +17,6 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { MatrixClient } from '../client'; import { - GROUP_CALL_ROOM_EVENT, - GROUP_CALL_MEMBER_EVENT, GroupCall, GroupCallIntent, GroupCallType, @@ -27,6 +25,7 @@ import { import { Room } from "../models/room"; import { RoomState } from "../models/room-state"; import { logger } from '../logger'; +import { EventType } from "../@types/event"; export class GroupCallEventHandler { public groupCalls = new Map(); // roomId -> GroupCall @@ -53,7 +52,7 @@ export class GroupCallEventHandler { } private createGroupCallForRoom(room: Room): GroupCall | undefined { - const callEvents = room.currentState.getStateEvents(GROUP_CALL_ROOM_EVENT); + const callEvents = room.currentState.getStateEvents(EventType.GroupCallPrefix); const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); for (const callEvent of sortedCallEvents) { @@ -125,7 +124,7 @@ export class GroupCallEventHandler { private onRoomStateChanged = (event: MatrixEvent, state: RoomState): void => { const eventType = event.getType(); - if (eventType === GROUP_CALL_ROOM_EVENT) { + if (eventType === EventType.GroupCallPrefix) { const groupCallId = event.getStateKey(); const content = event.getContent(); @@ -146,7 +145,7 @@ export class GroupCallEventHandler { logger.warn(`Multiple group calls detected for room: ${ state.roomId}. Multiple group calls are currently unsupported.`); } - } else if (eventType === GROUP_CALL_MEMBER_EVENT) { + } else if (eventType === EventType.GroupCallMemberPrefix) { const groupCall = this.groupCalls.get(state.roomId); if (!groupCall) { From e9b52e23d26c8df8d7661fcdfa95020ee60b7ab9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 19 Oct 2021 10:57:09 -0700 Subject: [PATCH 090/291] Rermove session id --- src/webrtc/groupCall.ts | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b3355480058..a257d614437 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -63,7 +63,6 @@ export interface IGroupCallDataChannelOptions { export interface IGroupCallRoomMemberCallState { "m.call_id": string; - "m.session_id": string; "m.foci"?: string[]; "m.sources"?: any[]; } @@ -107,8 +106,6 @@ export class GroupCall extends EventEmitter { public screenshareFeeds: CallFeed[] = []; public groupCallId: string; - private sessionId: string; - private sessionIds: Map = new Map(); private callHandlers: Map = new Map(); private activeSpeakerLoopTimeout?: number; private reEmitter: ReEmitter; @@ -124,7 +121,6 @@ export class GroupCall extends EventEmitter { super(); this.reEmitter = new ReEmitter(this); this.groupCallId = genCallID(); - this.sessionId = genCallID(); const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); @@ -487,7 +483,6 @@ export class GroupCall extends EventEmitter { private sendEnteredMemberStateEvent(): Promise { return this.updateMemberCallState({ "m.call_id": this.groupCallId, - "m.session_id": this.sessionId, // TODO "m.foci" // TODO "m.sources" }); @@ -573,13 +568,7 @@ export class GroupCall extends EventEmitter { return; } - const sessionId = callState["m.session_id"]; - - if (!sessionId || sessionId === this.sessionIds.get(member.userId)) { - return; - } - - this.addParticipant(member, sessionId); + this.addParticipant(member); if (this.state !== GroupCallState.Entered) { return; @@ -911,13 +900,12 @@ export class GroupCall extends EventEmitter { * Participant Management */ - private addParticipant(member: RoomMember, sessionId: string) { + private addParticipant(member: RoomMember) { if (this.participants.find((m) => m.userId === member.userId)) { return; } this.participants.push(member); - this.sessionIds.set(member.userId, sessionId); this.emit(GroupCallEvent.ParticipantsChanged, this.participants); this.client.emit("GroupCall.participants", this.participants, this); @@ -931,7 +919,6 @@ export class GroupCall extends EventEmitter { } this.participants.splice(index, 1); - this.sessionIds.delete(member.userId); this.emit(GroupCallEvent.ParticipantsChanged, this.participants); this.client.emit("GroupCall.participants", this.participants, this); From 8131b3900ddebc284b1a87e58937d649b58d1533 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 19 Oct 2021 15:30:20 -0700 Subject: [PATCH 091/291] Use glare resolution to manage group call setup --- src/webrtc/call.ts | 25 ++++++++----------------- src/webrtc/callEventHandler.ts | 2 +- src/webrtc/groupCall.ts | 27 ++++++++++++++++----------- 3 files changed, 25 insertions(+), 29 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index b515482c3d8..9f5244419ba 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -317,11 +317,6 @@ export class MatrixCall extends EventEmitter { private useToDevice: boolean; public groupCallId: string; - // Whether to stop local media when the call ends. We do not want to do this - // in group calls where the media is supplied by the group call. Doing this - // in a group call would result in breaking all the other calls - private stopLocalMediaOnEnd = true; - constructor(opts: CallOpts) { super(); this.roomId = opts.roomId; @@ -665,7 +660,7 @@ export class MatrixCall extends EventEmitter { private deleteAllFeeds(): void { for (const feed of this.feeds) { - if (!feed.isLocal() || this.stopLocalMediaOnEnd) { + if (!feed.isLocal() || !this.groupCallId) { feed.dispose(); } } @@ -838,7 +833,7 @@ export class MatrixCall extends EventEmitter { audioMuted: stream.getAudioTracks().length === 0, videoMuted: stream.getVideoTracks().length === 0, }); - this.answerWithCallFeeds([callFeed], true); + this.answerWithCallFeeds([callFeed]); } catch (e) { if (answerWithVideo) { // Try to answer without video @@ -856,13 +851,11 @@ export class MatrixCall extends EventEmitter { } } - public answerWithCallFeeds(callFeeds: CallFeed[], stopLocalMediaOnEnd?: boolean): void { + public answerWithCallFeeds(callFeeds: CallFeed[]): void { if (this.inviteOrAnswerSent) return; logger.debug(`Answering call ${this.callId}`); - this.stopLocalMediaOnEnd = stopLocalMediaOnEnd; - this.gotCallFeedsForAnswer(callFeeds); } @@ -2055,22 +2048,21 @@ export class MatrixCall extends EventEmitter { } private stopAllMedia(): void { - logger.debug(this.stopLocalMediaOnEnd ? "Stopping all media" : "Stopping all media except local feeds" ); - + logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); for (const feed of this.feeds) { if ( feed.isLocal() && feed.purpose === SDPStreamMetadataPurpose.Usermedia && - this.stopLocalMediaOnEnd + !this.groupCallId ) { this.client.getMediaHandler().stopUserMediaStream(feed.stream); } else if ( feed.isLocal() && feed.purpose === SDPStreamMetadataPurpose.Screenshare && - this.stopLocalMediaOnEnd + !this.groupCallId ) { this.client.getMediaHandler().stopScreensharingStream(feed.stream); - } else if (!feed.isLocal() || this.stopLocalMediaOnEnd) { + } else if (!feed.isLocal() || !this.groupCallId) { for (const track of feed.stream.getTracks()) { track.stop(); } @@ -2170,9 +2162,8 @@ export class MatrixCall extends EventEmitter { * @throws if you have not specified a listener for 'error' events. * @throws if have passed audio=false. */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[], stopLocalMediaOnEnd?: boolean): Promise { + public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise { this.checkForErrorListener(); - this.stopLocalMediaOnEnd = stopLocalMediaOnEnd; this.direction = CallDirection.Outbound; // XXX Find a better way to do this diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 2454c1dafd3..ae1ab59fe01 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -231,7 +231,7 @@ export class CallEventHandler { if ( call.roomId === thisCall.roomId && thisCall.direction === CallDirection.Outbound && - call.invitee === thisCall.invitee && + call.getOpponentMember().userId === thisCall.invitee && isCalling ) { existingCall = thisCall; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a257d614437..6bbf4db2177 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -84,6 +84,7 @@ interface ICallHandlers { onCallFeedsChanged: (feeds: CallFeed[]) => void; onCallStateChanged: (state: CallState, oldState: CallState) => void; onCallHangup: (call: MatrixCall) => void; + onCallReplaced: (newCall: MatrixCall) => void; } function getCallUserId(call: MatrixCall): string | null { @@ -464,6 +465,10 @@ export class GroupCall extends EventEmitter { const opponentMemberId = newCall.getOpponentMember().userId; const existingCall = this.getCallByUserId(opponentMemberId); + if (existingCall && existingCall.callId === newCall.callId) { + return; + } + logger.log(`GroupCall: incoming call from: ${opponentMemberId}`); // Check if the user calling has an existing call and use this call instead. @@ -574,15 +579,12 @@ export class GroupCall extends EventEmitter { return; } - // Only initiate a call with a user who has a userId that is lexicographically - // less than your own. Otherwise, that user will call you. - if (member.userId < localUserId) { - logger.log(`Waiting for ${member.userId} to send call invite.`); + const existingCall = this.getCallByUserId(member.userId); + + if (existingCall) { return; } - const existingCall = this.getCallByUserId(member.userId); - const newCall = createNewMatrixCall( this.client, this.room.roomId, @@ -595,11 +597,7 @@ export class GroupCall extends EventEmitter { newCall.createDataChannel("datachannel", this.dataChannelOptions); } - if (existingCall) { - this.replaceCall(existingCall, newCall); - } else { - this.addCall(newCall); - } + this.addCall(newCall); }; /** @@ -656,18 +654,23 @@ export class GroupCall extends EventEmitter { const onCallStateChanged = (state: CallState, oldState: CallState) => this.onCallStateChanged(call, state, oldState); const onCallHangup = this.onCallHangup; + const onCallReplaced = (newCall: MatrixCall) => this.replaceCall(call, newCall); this.callHandlers.set(opponentMemberId, { onCallFeedsChanged, onCallStateChanged, onCallHangup, + onCallReplaced, }); call.on(CallEvent.FeedsChanged, onCallFeedsChanged); call.on(CallEvent.State, onCallStateChanged); call.on(CallEvent.Hangup, onCallHangup); + call.on(CallEvent.Replaced, onCallReplaced); this.reEmitter.reEmit(call, Object.values(CallEvent)); + + onCallFeedsChanged(); } private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { @@ -681,11 +684,13 @@ export class GroupCall extends EventEmitter { onCallFeedsChanged, onCallStateChanged, onCallHangup, + onCallReplaced, } = this.callHandlers.get(opponentMemberId); call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); call.removeListener(CallEvent.State, onCallStateChanged); call.removeListener(CallEvent.Hangup, onCallHangup); + call.removeListener(CallEvent.Replaced, onCallReplaced); this.callHandlers.delete(opponentMemberId); From 159e825877d626179691d9b919c57eaebf3faf16 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 19 Oct 2021 15:31:59 -0700 Subject: [PATCH 092/291] Fix unnecessary param to placeCallWithCallFeeds --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9f5244419ba..ae25deafd12 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2149,7 +2149,7 @@ export class MatrixCall extends EventEmitter { audioMuted: stream.getAudioTracks().length === 0, videoMuted: stream.getVideoTracks().length === 0, }); - await this.placeCallWithCallFeeds([callFeed], true); + await this.placeCallWithCallFeeds([callFeed]); } catch (e) { this.getUserMediaFailed(e); return; From 0555f9db1c19b70507b8dcb2962d7ad7e9f291c9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 19 Oct 2021 17:05:21 -0700 Subject: [PATCH 093/291] Only send to device messages to a single device --- src/webrtc/call.ts | 12 ++++----- src/webrtc/callEventHandler.ts | 29 ++++++++++++++++++---- src/webrtc/groupCall.ts | 45 +++++++++++++++++++++++++++++++--- 3 files changed, 72 insertions(+), 14 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ae25deafd12..59560081913 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -71,7 +71,7 @@ interface CallOpts { client?: any; // Fix when client is TSified forceTURN?: boolean; turnServers?: Array; - useToDevice?: boolean; + opponentDeviceId?: string; groupCallId?: string; } @@ -314,7 +314,7 @@ export class MatrixCall extends EventEmitter { private callLengthInterval: number; private callLength = 0; - private useToDevice: boolean; + private opponentDeviceId: string; public groupCallId: string; constructor(opts: CallOpts) { @@ -324,7 +324,7 @@ export class MatrixCall extends EventEmitter { this.client = opts.client; this.forceTURN = opts.forceTURN; this.ourPartyId = this.client.deviceId; - this.useToDevice = opts.useToDevice; + this.opponentDeviceId = opts.opponentDeviceId; this.groupCallId = opts.groupCallId; // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; @@ -1910,10 +1910,10 @@ export class MatrixCall extends EventEmitter { conf_id: this.groupCallId, }); - if (this.useToDevice) { + if (this.opponentDeviceId) { return this.client.sendToDevice(eventType, { [this.invitee || this.getOpponentMember().userId]: { - "*": realContent, + [this.opponentDeviceId]: realContent, }, }); } else { @@ -2326,7 +2326,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO turnServers: client.getTurnServers(), // call level options forceTURN: client.forceTURN || optionsForceTURN, - useToDevice: options?.useToDevice, + opponentDeviceId: options?.opponentDeviceId, groupCallId: options?.groupCallId, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index ae1ab59fe01..65ad4f52951 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -141,7 +141,7 @@ export class CallEventHandler { for (const event of this.toDeviceCallEventBuffer) { try { - await this.handleCallEvent(event, true); + await this.handleCallEvent(event); } catch (e) { logger.error("Caught exception handling call event", e); } @@ -159,7 +159,7 @@ export class CallEventHandler { return type.startsWith("m.call.") || type.startsWith("org.matrix.call."); } - private async handleCallEvent(event: MatrixEvent, isToDevice?: boolean) { + private async handleCallEvent(event: MatrixEvent) { const content = event.getContent(); const callRoomId = ( event.getRoomId() || @@ -167,10 +167,29 @@ export class CallEventHandler { ); const groupCallId = content.conf_id; const type = event.getType() as EventType; - const weSentTheEvent = event.getSender() === this.client.credentials.userId; + const senderId = event.getSender(); + const weSentTheEvent = senderId === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; //console.info("RECV %s content=%s", type, JSON.stringify(content)); + let opponentDeviceId: string | undefined; + + if (groupCallId) { + const groupCall = this.client.groupCallEventHandler.getGroupCallById(groupCallId); + + if (!groupCall) { + logger.warn(`Cannot find a group call ${groupCallId} for event ${type}. Ignoring event.`); + return; + } + + opponentDeviceId = groupCall.getDeviceIdForMember(senderId); + + if (!opponentDeviceId) { + logger.warn(`Cannot find a device id for ${senderId}. Ignoring event.`); + return; + } + } + if (!callRoomId) return; if (type === EventType.CallInvite) { @@ -197,7 +216,7 @@ export class CallEventHandler { call = createNewMatrixCall( this.client, callRoomId, - { forceTURN: this.client.forceTURN, useToDevice: isToDevice, groupCallId }, + { forceTURN: this.client.forceTURN, opponentDeviceId, groupCallId }, ); if (!call) { logger.log( @@ -287,7 +306,7 @@ export class CallEventHandler { // we're probably getting events backwards so // the hangup will come before the invite call = createNewMatrixCall( - this.client, callRoomId, { useToDevice: isToDevice }, + this.client, callRoomId, { opponentDeviceId }, ); if (call) { call.callId = content.call_id; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 6bbf4db2177..a48c92d30cd 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -61,10 +61,14 @@ export interface IGroupCallDataChannelOptions { protocol: string; } +export interface IGroupCallRoomMemberSource { + "device_id": string; +} + export interface IGroupCallRoomMemberCallState { "m.call_id": string; "m.foci"?: string[]; - "m.sources"?: any[]; + "m.sources": IGroupCallRoomMemberSource[]; } export interface IGroupCallRoomMemberState { @@ -488,8 +492,13 @@ export class GroupCall extends EventEmitter { private sendEnteredMemberStateEvent(): Promise { return this.updateMemberCallState({ "m.call_id": this.groupCallId, + "m.sources": [ + { + "device_id": this.client.getDeviceId(), + // TODO rest of the source properties + }, + ], // TODO "m.foci" - // TODO "m.sources" }); } @@ -585,10 +594,17 @@ export class GroupCall extends EventEmitter { return; } + const opponentDeviceId = this.getDeviceIdForMember(member.userId); + + if (!opponentDeviceId) { + logger.warn(`No opponent device id found for ${member.userId}, ignoring.`); + return; + } + const newCall = createNewMatrixCall( this.client, this.room.roomId, - { invitee: member.userId, useToDevice: true, groupCallId: this.groupCallId }, + { invitee: member.userId, opponentDeviceId, groupCallId: this.groupCallId }, ); newCall.placeCallWithCallFeeds(this.getLocalFeeds()); @@ -600,6 +616,29 @@ export class GroupCall extends EventEmitter { this.addCall(newCall); }; + public getDeviceIdForMember(userId: string): string | undefined { + const memberStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); + + if (!memberStateEvent) { + return undefined; + } + + const memberState = memberStateEvent.getContent(); + const memberGroupCallState = memberState["m.calls"]?.find((call) => call["m.call_id"] === this.groupCallId); + + if (!memberGroupCallState) { + return undefined; + } + + const memberSources = memberGroupCallState["m.sources"]; + + if (!memberSources || memberSources.length === 0) { + return undefined; + } + + return memberSources[0].device_id; + } + /** * Call Event Handlers */ From 305de54106e325a9287fe89501cfd6c397e0c784 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 21 Oct 2021 12:57:30 -0700 Subject: [PATCH 094/291] Fix screensharing and webrtc races --- src/webrtc/call.ts | 47 +++++++++++++++-------------- src/webrtc/callEventHandler.ts | 2 +- src/webrtc/groupCall.ts | 54 ++++++++++++++++++++++------------ 3 files changed, 62 insertions(+), 41 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 59560081913..f9b0d5e60bf 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -824,7 +824,7 @@ export class MatrixCall extends EventEmitter { answerWithAudio, answerWithVideo, ); this.waitForLocalAVStream = false; - const callFeed = new CallFeed({ + const usermediaFeed = new CallFeed({ client: this.client, roomId: this.roomId, userId: this.client.getUserId(), @@ -833,7 +833,14 @@ export class MatrixCall extends EventEmitter { audioMuted: stream.getAudioTracks().length === 0, videoMuted: stream.getVideoTracks().length === 0, }); - this.answerWithCallFeeds([callFeed]); + + const feeds = [usermediaFeed]; + + if (this.localScreensharingFeed) { + feeds.push(this.localScreensharingFeed); + } + + this.answerWithCallFeeds(feeds); } catch (e) { if (answerWithVideo) { // Try to answer without video @@ -1262,7 +1269,7 @@ export class MatrixCall extends EventEmitter { setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); } - private gotCallFeedsForInvite(callFeeds: CallFeed[]): void { + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { if (this.successor) { this.successor.gotCallFeedsForAnswer(callFeeds); return; @@ -1275,6 +1282,13 @@ export class MatrixCall extends EventEmitter { for (const feed of callFeeds) { this.pushLocalFeed(feed); } + + if (requestScreenshareFeed) { + this.peerConn.addTransceiver("video", { + direction: "recvonly", + }); + } + this.setState(CallState.CreateOffer); logger.debug("gotUserMediaForInvite"); @@ -1339,18 +1353,9 @@ export class MatrixCall extends EventEmitter { this.setState(CallState.CreateAnswer); - let myAnswer; try { this.getRidOfRTXCodecs(); - myAnswer = await this.peerConn.createAnswer(); - } catch (err) { - logger.debug("Failed to create answer: ", err); - this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); - return; - } - - try { - await this.peerConn.setLocalDescription(myAnswer); + await this.peerConn.setLocalDescription(); this.setState(CallState.Connecting); // Allow a short time for initial candidates to be gathered @@ -1559,8 +1564,7 @@ export class MatrixCall extends EventEmitter { if (description.type === 'offer') { this.getRidOfRTXCodecs(); - const localDescription = await this.peerConn.createAnswer(); - await this.peerConn.setLocalDescription(localDescription); + await this.peerConn.setLocalDescription(); this.sendVoipEvent(EventType.CallNegotiate, { description: this.peerConn.localDescription, @@ -1613,8 +1617,8 @@ export class MatrixCall extends EventEmitter { return this.state === CallState.Ended; } - private gotLocalOffer = async (description: RTCSessionDescriptionInit): Promise => { - logger.debug("Created offer: ", description); + private gotLocalOffer = async (): Promise => { + logger.debug("Setting local description"); if (this.callHasEnded()) { logger.debug("Ignoring newly created offer on call ID " + this.callId + @@ -1623,7 +1627,7 @@ export class MatrixCall extends EventEmitter { } try { - await this.peerConn.setLocalDescription(description); + await this.peerConn.setLocalDescription(); } catch (err) { logger.debug("Error setting local description!", err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); @@ -1840,8 +1844,7 @@ export class MatrixCall extends EventEmitter { this.makingOffer = true; try { this.getRidOfRTXCodecs(); - const myOffer = await this.peerConn.createOffer(); - await this.gotLocalOffer(myOffer); + await this.gotLocalOffer(); } catch (e) { this.getLocalOfferFailed(e); return; @@ -2162,7 +2165,7 @@ export class MatrixCall extends EventEmitter { * @throws if you have not specified a listener for 'error' events. * @throws if have passed audio=false. */ - public async placeCallWithCallFeeds(callFeeds: CallFeed[]): Promise { + public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { this.checkForErrorListener(); this.direction = CallDirection.Outbound; @@ -2179,7 +2182,7 @@ export class MatrixCall extends EventEmitter { // create the peer connection now so it can be gathering candidates while we get user // media (assuming a candidate pool size is configured) this.peerConn = this.createPeerConnection(); - this.gotCallFeedsForInvite(callFeeds); + this.gotCallFeedsForInvite(callFeeds, requestScreenshareFeed); } private createPeerConnection(): RTCPeerConnection { diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 65ad4f52951..3bf66e47fae 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -182,7 +182,7 @@ export class CallEventHandler { return; } - opponentDeviceId = groupCall.getDeviceIdForMember(senderId); + opponentDeviceId = groupCall.getDeviceForMember(senderId)?.device_id; if (!opponentDeviceId) { logger.warn(`Cannot find a device id for ${senderId}. Ignoring event.`); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a48c92d30cd..8e058e82c43 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -61,14 +61,20 @@ export interface IGroupCallDataChannelOptions { protocol: string; } -export interface IGroupCallRoomMemberSource { +export interface IGroupCallRoomMemberFeed { + purpose: SDPStreamMetadataPurpose; + // TODO: Sources for adaptive bitrate +} + +export interface IGroupCallRoomMemberDevice { "device_id": string; + "feeds": IGroupCallRoomMemberFeed[]; } export interface IGroupCallRoomMemberCallState { "m.call_id": string; "m.foci"?: string[]; - "m.sources": IGroupCallRoomMemberSource[]; + "m.devices": IGroupCallRoomMemberDevice[]; } export interface IGroupCallRoomMemberState { @@ -225,7 +231,7 @@ export class GroupCall extends EventEmitter { await this.initLocalCallFeed(); } - this.sendEnteredMemberStateEvent(); + this.sendMemberStateEvent(); this.activeSpeaker = null; @@ -276,7 +282,7 @@ export class GroupCall extends EventEmitter { return; } - this.sendLeftMemberStateEvent(); + this.removeMemberStateEvent(); while (this.calls.length > 0) { this.removeCall(this.calls[this.calls.length - 1], CallErrorCode.UserHangup); @@ -414,6 +420,8 @@ export class GroupCall extends EventEmitter { // TODO: handle errors await Promise.all(this.calls.map(call => call.pushLocalFeed(this.localScreenshareFeed))); + await this.sendMemberStateEvent(); + logger.log("screensharing enabled on all calls"); return true; @@ -430,6 +438,7 @@ export class GroupCall extends EventEmitter { this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; + await this.sendMemberStateEvent(); this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); return false; } @@ -489,20 +498,25 @@ export class GroupCall extends EventEmitter { * Room Member State */ - private sendEnteredMemberStateEvent(): Promise { + private sendMemberStateEvent(): Promise { + const deviceId = this.client.getDeviceId(); + return this.updateMemberCallState({ "m.call_id": this.groupCallId, - "m.sources": [ + "m.devices": [ { - "device_id": this.client.getDeviceId(), - // TODO rest of the source properties + "device_id": deviceId, + "feeds": this.getLocalFeeds().map((feed) => ({ + purpose: feed.purpose, + })), + // TODO: Add data channels }, ], // TODO "m.foci" }); } - private sendLeftMemberStateEvent(): Promise { + private removeMemberStateEvent(): Promise { return this.updateMemberCallState(undefined); } @@ -594,20 +608,23 @@ export class GroupCall extends EventEmitter { return; } - const opponentDeviceId = this.getDeviceIdForMember(member.userId); + const opponentDevice = this.getDeviceForMember(member.userId); - if (!opponentDeviceId) { - logger.warn(`No opponent device id found for ${member.userId}, ignoring.`); + if (!opponentDevice) { + logger.warn(`No opponent device found for ${member.userId}, ignoring.`); return; } const newCall = createNewMatrixCall( this.client, this.room.roomId, - { invitee: member.userId, opponentDeviceId, groupCallId: this.groupCallId }, + { invitee: member.userId, opponentDeviceId: opponentDevice.device_id, groupCallId: this.groupCallId }, ); - newCall.placeCallWithCallFeeds(this.getLocalFeeds()); + const requestScreenshareFeed = opponentDevice.feeds.some( + (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + + newCall.placeCallWithCallFeeds(this.getLocalFeeds(), requestScreenshareFeed); if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); @@ -616,7 +633,7 @@ export class GroupCall extends EventEmitter { this.addCall(newCall); }; - public getDeviceIdForMember(userId: string): string | undefined { + public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { const memberStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); if (!memberStateEvent) { @@ -630,13 +647,14 @@ export class GroupCall extends EventEmitter { return undefined; } - const memberSources = memberGroupCallState["m.sources"]; + const memberDevices = memberGroupCallState["m.devices"]; - if (!memberSources || memberSources.length === 0) { + if (!memberDevices || memberDevices.length === 0) { return undefined; } - return memberSources[0].device_id; + // NOTE: For now we only support one device so we use the device id in the first source. + return memberDevices[0]; } /** From 3b0d1b269605dc7343ca23017d268251ea25ec06 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 21 Oct 2021 14:23:27 -0700 Subject: [PATCH 095/291] Add check for existing group call session --- src/client.ts | 1 + src/webrtc/groupCall.ts | 30 ++++++++++++++++++++--------- src/webrtc/groupCallEventHandler.ts | 2 +- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/src/client.ts b/src/client.ts index e8d4f0114a9..bce3f530227 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1332,6 +1332,7 @@ export class MatrixClient extends EventEmitter { room, type, intent, + undefined, dataChannelsEnabled, dataChannelOptions, ).create(); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 8e058e82c43..613516c8e25 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -126,12 +126,13 @@ export class GroupCall extends EventEmitter { public room: Room, public type: GroupCallType, public intent: GroupCallIntent, + groupCallId?: string, private dataChannelsEnabled?: boolean, private dataChannelOptions?: IGroupCallDataChannelOptions, ) { super(); this.reEmitter = new ReEmitter(this); - this.groupCallId = genCallID(); + this.groupCallId = groupCallId || genCallID(); const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); @@ -177,6 +178,11 @@ export class GroupCall extends EventEmitter { return feeds; } + public hasLocalParticipant(): boolean { + const userId = this.client.getUserId(); + return this.participants.some((member) => member.userId === userId); + } + public async initLocalCallFeed(): Promise { if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); @@ -231,6 +237,8 @@ export class GroupCall extends EventEmitter { await this.initLocalCallFeed(); } + this.addParticipant(this.room.getMember(this.client.getUserId())); + this.sendMemberStateEvent(); this.activeSpeaker = null; @@ -282,6 +290,8 @@ export class GroupCall extends EventEmitter { return; } + this.removeParticipant(this.room.getMember(this.client.getUserId())); + this.removeMemberStateEvent(); while (this.calls.length > 0) { @@ -564,13 +574,6 @@ export class GroupCall extends EventEmitter { return; } - // Don't process your own member. - const localUserId = this.client.getUserId(); - - if (member.userId === localUserId) { - return; - } - const callsState = event.getContent()["m.calls"]; if (!callsState || !Array.isArray(callsState) || callsState.length === 0) { @@ -579,6 +582,8 @@ export class GroupCall extends EventEmitter { return; } + logger.log(callsState); + // Currently we only support a single call per room. So grab the first call. const callState = callsState[0]; @@ -591,13 +596,20 @@ export class GroupCall extends EventEmitter { } if (callId !== this.groupCallId) { - logger.log(`Call id does not match group call id, ignoring.`); + logger.log(`Call id ${callId} does not match group call id ${this.groupCallId}, ignoring.`); this.removeParticipant(member); return; } this.addParticipant(member); + // Don't process your own member. + const localUserId = this.client.getUserId(); + + if (member.userId === localUserId) { + return; + } + if (this.state !== GroupCallState.Entered) { return; } diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index fd76299b0ea..c8d7485b15d 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -106,10 +106,10 @@ export class GroupCallEventHandler { room, callType, callIntent, + groupCallId, content?.dataChannelsEnabled, dataChannelOptions, ); - groupCall.groupCallId = groupCallId; this.groupCalls.set(room.roomId, groupCall); this.client.emit("GroupCall.incoming", groupCall); From b4d8c0b6030180366b1dc2cd084ee7c19cf899fc Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 22 Oct 2021 11:27:02 -0700 Subject: [PATCH 096/291] Fix updating member state with no existing calls --- src/webrtc/groupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 613516c8e25..57981759ba1 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -538,11 +538,11 @@ export class GroupCall extends EventEmitter { let calls: IGroupCallRoomMemberCallState[] = []; let existingCallIndex: number; - if (memberCallState) { + if (memberStateEvent) { calls = memberStateEvent["m.calls"] || []; existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); } else { - existingCallIndex = 0; + existingCallIndex = -1; } if (existingCallIndex === -1) { From fc8a867e8eda55a7fd59e6d77ffecdf29f851e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 25 Oct 2021 23:02:28 +0200 Subject: [PATCH 097/291] Start processing member state events only after we've set out own (#2000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This avoids a race condition where the other side would first receive the to-device messages and only then the member state event which would result in the call being ignored Signed-off-by: Šimon Brandner --- src/webrtc/groupCall.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 57981759ba1..1f1f861993d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -239,7 +239,7 @@ export class GroupCall extends EventEmitter { this.addParticipant(this.room.getMember(this.client.getUserId())); - this.sendMemberStateEvent(); + const sendMemberStateEventPromise = this.sendMemberStateEvent(); this.activeSpeaker = null; @@ -262,9 +262,14 @@ export class GroupCall extends EventEmitter { logger.log("Processing initial members"); - for (const stateEvent of memberStateEvents) { - this.onMemberStateChanged(stateEvent); - } + // This avoids a race condition where the other side would first receive + // the to-device messages and only then the member state event which + // would result in the call being ignored + sendMemberStateEventPromise.then(() => { + for (const stateEvent of memberStateEvents) { + this.onMemberStateChanged(stateEvent); + } + }); this.client.on("Call.incoming", this.onIncomingCall); From 119ce2e46f196612a335900c930427ec0a88e306 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 26 Oct 2021 12:44:37 -0700 Subject: [PATCH 098/291] Fix inbound calls in Safari --- src/webrtc/call.ts | 31 +++++++++----- src/webrtc/callFeed.ts | 21 +++++++++ src/webrtc/groupCall.ts | 14 +++--- src/webrtc/mediaHandler.ts | 87 ++++++++++++++++++++------------------ 4 files changed, 96 insertions(+), 57 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f9b0d5e60bf..c0caf42c7c4 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -228,6 +228,8 @@ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; +const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + export class CallError extends Error { code: string; @@ -877,7 +879,7 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { logger.debug("Handing local stream to new call"); - newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); + newCall.gotCallFeedsForAnswer(this.getLocalFeeds().map((feed) => feed.clone())); } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -1085,15 +1087,10 @@ export class MatrixCall extends EventEmitter { /** * Request a new local usermedia stream with the current device id. */ - public async updateLocalUsermediaStream() { - const stream = await this.client.getMediaHandler().getUserMediaStream( - this.hasLocalUserMediaAudioTrack, - this.hasLocalUserMediaVideoTrack, - true, - ); + public async updateLocalUsermediaStream(stream: MediaStream) { + const oldStream = this.localUsermediaStream; const callFeed = this.localUsermediaFeed; - callFeed.setNewStream(stream); const newSenders = []; @@ -1110,7 +1107,7 @@ export class MatrixCall extends EventEmitter { `Replacing track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + - `streamId="${stream}", ` + + `streamId="${stream.id}", ` + `streamPurpose="${callFeed.purpose}"` + `) to peer connection`, ); @@ -1121,7 +1118,7 @@ export class MatrixCall extends EventEmitter { `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + - `streamId="${stream}", ` + + `streamId="${stream.id}", ` + `streamPurpose="${callFeed.purpose}"` + `) to peer connection`, ); @@ -1132,6 +1129,8 @@ export class MatrixCall extends EventEmitter { } this.usermediaSenders = newSenders; + + this.client.getMediaHandler().stopUserMediaStream(oldStream); } /** @@ -1364,6 +1363,18 @@ export class MatrixCall extends EventEmitter { }); this.sendAnswer(); + + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); + + await this.client.getMediaHandler().updateLocalUsermediaStreams(); + } } catch (err) { logger.debug("Error setting local description!", err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 1636540c668..39dff55dcae 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -250,6 +250,27 @@ export class CallFeed extends EventEmitter { this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); }; + public clone(): CallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this.stream.clone(); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } + + return new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.userId, + stream, + purpose: this.purpose, + audioMuted: this.audioMuted, + videoMuted: this.videoMuted, + }); + } + public dispose(): void { clearTimeout(this.volumeLooperTimeout); } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 1f1f861993d..d38db94847f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -219,11 +219,11 @@ export class GroupCall extends EventEmitter { return callFeed; } - public async updateLocalUsermediaStream() { + public async updateLocalUsermediaStream(stream: MediaStream) { if (this.localCallFeed) { - const mediaHandler = this.client.getMediaHandler(); - const stream = await mediaHandler.getUserMediaStream(true, this.type === GroupCallType.Video, true); + const oldStream = this.localCallFeed.stream; this.localCallFeed.setNewStream(stream); + this.client.getMediaHandler().stopUserMediaStream(oldStream); } } @@ -433,7 +433,7 @@ export class GroupCall extends EventEmitter { ); // TODO: handle errors - await Promise.all(this.calls.map(call => call.pushLocalFeed(this.localScreenshareFeed))); + await Promise.all(this.calls.map(call => call.pushLocalFeed(this.localScreenshareFeed.clone()))); await this.sendMemberStateEvent(); @@ -506,7 +506,8 @@ export class GroupCall extends EventEmitter { this.addCall(newCall); } - newCall.answerWithCallFeeds(this.getLocalFeeds()); + // Safari can't send a MediaStream to multiple sources, so clone it + newCall.answerWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); }; /** @@ -641,7 +642,8 @@ export class GroupCall extends EventEmitter { const requestScreenshareFeed = opponentDevice.feeds.some( (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - newCall.placeCallWithCallFeeds(this.getLocalFeeds(), requestScreenshareFeed); + // Safari can't send a MediaStream to multiple sources, so clone it + newCall.placeCallWithCallFeeds(this.getLocalFeeds().map(feed => feed.clone()), requestScreenshareFeed); if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 38308270739..597c766b832 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -17,14 +17,17 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { GroupCallType } from "../webrtc/groupCall"; import { MatrixClient } from "../client"; import { logger } from "../logger"; +import { CallState } from "./call"; export class MediaHandler { private audioInput: string; private videoInput: string; - private userMediaStreams: MediaStream[] = []; - private screensharingStreams: MediaStream[] = []; + private localUserMediaStream?: MediaStream; + public userMediaStreams: MediaStream[] = []; + public screensharingStreams: MediaStream[] = []; constructor(private client: MatrixClient) {} @@ -35,20 +38,7 @@ export class MediaHandler { */ public async setAudioInput(deviceId: string): Promise { this.audioInput = deviceId; - - for (const stream of this.userMediaStreams) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - - await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { - return call.updateLocalUsermediaStream(); - })); - - await Promise.all(Array.from(this.client.groupCallEventHandler.groupCalls.values()).map((groupCall) => { - return groupCall.updateLocalUsermediaStream(); - })); + await this.updateLocalUsermediaStreams(); } /** @@ -58,20 +48,26 @@ export class MediaHandler { */ public async setVideoInput(deviceId: string): Promise { this.videoInput = deviceId; + await this.updateLocalUsermediaStreams(); + } - for (const stream of this.userMediaStreams) { - for (const track of stream.getTracks()) { - track.stop(); - } - } - - await Promise.all(Array.from(this.client.callEventHandler.calls.values()).map((call) => { - return call.updateLocalUsermediaStream(); - })); - - await Promise.all(Array.from(this.client.groupCallEventHandler.groupCalls.values()).map((groupCall) => { - return groupCall.updateLocalUsermediaStream(); - })); + public async updateLocalUsermediaStreams(): Promise { + this.localUserMediaStream = undefined; + + await Promise.all(Array.from(this.client.callEventHandler.calls.values()) + .filter((call) => call.state !== CallState.Ended) + .map((call) => { + return this.getUserMediaStream( + call.hasLocalUserMediaAudioTrack, + call.hasLocalUserMediaVideoTrack, + ).then(stream => call.updateLocalUsermediaStream(stream)); + })); + + await Promise.all(Array.from(this.client.groupCallEventHandler.groupCalls.values()) + .map((groupCall) => { + return this.getUserMediaStream(true, groupCall.type === GroupCallType.Video) + .then(stream => groupCall.updateLocalUsermediaStream(stream)); + })); } public async hasAudioDevice(): Promise { @@ -87,26 +83,35 @@ export class MediaHandler { /** * @returns {MediaStream} based on passed parameters */ - public async getUserMediaStream(audio: boolean, video: boolean, forceNewStream = false): Promise { + public async getUserMediaStream(audio: boolean, video: boolean): Promise { const shouldRequestAudio = audio && await this.hasAudioDevice(); const shouldRequestVideo = video && await this.hasVideoDevice(); let stream: MediaStream; - // Find a stream with matching tracks - const matchingStream = this.userMediaStreams.find((stream) => { - if (shouldRequestAudio !== (stream.getAudioTracks().length > 0)) return false; - if (shouldRequestVideo !== (stream.getVideoTracks().length > 0)) return false; - return true; - }); - - if (matchingStream && !forceNewStream) { - logger.log("Cloning user media stream", matchingStream.id); - stream = matchingStream.clone(); - } else { + if ( + !this.localUserMediaStream || + (this.localUserMediaStream.getAudioTracks().length === 0 && shouldRequestAudio) || + (this.localUserMediaStream.getVideoTracks().length === 0 && shouldRequestVideo) + ) { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); logger.log("Getting user media with constraints", constraints); stream = await navigator.mediaDevices.getUserMedia(constraints); + this.localUserMediaStream = stream; + } else { + stream = this.localUserMediaStream.clone(); + + if (!shouldRequestAudio) { + for (const track of stream.getAudioTracks()) { + stream.removeTrack(track); + } + } + + if (!shouldRequestVideo) { + for (const track of stream.getVideoTracks()) { + stream.removeTrack(track); + } + } } this.userMediaStreams.push(stream); From 9f3f9990ef790a3648941de003ca842d3c7d6575 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 18 Oct 2021 22:41:01 +0100 Subject: [PATCH 099/291] untested first cut at factoring out a encryptAndSendToDevices method --- src/crypto/algorithms/megolm.ts | 82 +++++--------------------------- src/crypto/index.ts | 84 +++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 71 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index 167c811300c..f771674f5b8 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -584,81 +584,21 @@ class MegolmEncryption extends EncryptionAlgorithm { userDeviceMap: IOlmDevice[], payload: IPayload, ): Promise { - const contentMap = {}; - const deviceInfoByDeviceId = new Map(); - - const promises = []; - for (let i = 0; i < userDeviceMap.length; i++) { - const encryptedContent = { - algorithm: olmlib.OLM_ALGORITHM, - sender_key: this.olmDevice.deviceCurve25519Key, - ciphertext: {}, - }; - const val = userDeviceMap[i]; - const userId = val.userId; - const deviceInfo = val.deviceInfo; - const deviceId = deviceInfo.deviceId; - deviceInfoByDeviceId.set(deviceId, deviceInfo); - - if (!contentMap[userId]) { - contentMap[userId] = {}; - } - contentMap[userId][deviceId] = encryptedContent; - - promises.push( - olmlib.encryptMessageForDevice( - encryptedContent.ciphertext, - this.userId, - this.deviceId, - this.olmDevice, - userId, - deviceInfo, - payload, - ), - ); - } - - return Promise.all(promises).then(() => { - // prune out any devices that encryptMessageForDevice could not encrypt for, - // in which case it will have just not added anything to the ciphertext object. - // There's no point sending messages to devices if we couldn't encrypt to them, - // since that's effectively a blank message. + this.crypto.encryptAndSendToDevices( + userDeviceMap, + payload + ).then(() => { + // store that we successfully uploaded the keys of the current slice for (const userId of Object.keys(contentMap)) { for (const deviceId of Object.keys(contentMap[userId])) { - if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) { - logger.log( - "No ciphertext for device " + - userId + ":" + deviceId + ": pruning", - ); - delete contentMap[userId][deviceId]; - } - } - // No devices left for that user? Strip that too. - if (Object.keys(contentMap[userId]).length === 0) { - logger.log("Pruned all devices for user " + userId); - delete contentMap[userId]; + session.markSharedWithDevice( + userId, + deviceId, + deviceInfoByDeviceId.get(deviceId).getIdentityKey(), + chainIndex, + ); } } - - // Is there anything left? - if (Object.keys(contentMap).length === 0) { - logger.log("No users left to send to: aborting"); - return; - } - - return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then(() => { - // store that we successfully uploaded the keys of the current slice - for (const userId of Object.keys(contentMap)) { - for (const deviceId of Object.keys(contentMap[userId])) { - session.markSharedWithDevice( - userId, - deviceId, - deviceInfoByDeviceId.get(deviceId).getIdentityKey(), - chainIndex, - ); - } - } - }); }); } diff --git a/src/crypto/index.ts b/src/crypto/index.ts index d10ba9ac27a..94390993ed3 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3030,6 +3030,90 @@ export class Crypto extends EventEmitter { }); } + /** + * @private + * Encrypts and sends a given object via Olm to-device message to a given + * set of devices. Heavily derived from encryptAndSendKeysToDevices in + * megolm.ts. + * + * @param {object} userDeviceMap + * mapping from userId to deviceInfo + * + * @param {object} payload fields to include in the encrypted payload + * + * @return {Promise} Promise which resolves once the key sharing + * for the given userDeviceMap is generated and has been sent. + */ + private encryptAndSendToDevices( + userDeviceMap: IExportedDevice[], + payload: object, + ): Promise<{}> { + const contentMap = {}; + const deviceInfoByDeviceId = new Map(); + + const promises = []; + for (let i = 0; i < userDeviceMap.length; i++) { + const encryptedContent = { + algorithm: olmlib.OLM_ALGORITHM, + sender_key: this.olmDevice.deviceCurve25519Key, + ciphertext: {}, + }; + const val = userDeviceMap[i]; + const userId = val.userId; + const deviceInfo = val.deviceInfo; + const deviceId = deviceInfo.deviceId; + deviceInfoByDeviceId.set(deviceId, deviceInfo); + + if (!contentMap[userId]) { + contentMap[userId] = {}; + } + contentMap[userId][deviceId] = encryptedContent; + + promises.push( + olmlib.encryptMessageForDevice( + encryptedContent.ciphertext, + this.userId, + this.deviceId, + this.olmDevice, + userId, + deviceInfo, + payload, + ), + ); + } + + return Promise.all(promises).then(() => { + // prune out any devices that encryptMessageForDevice could not encrypt for, + // in which case it will have just not added anything to the ciphertext object. + // There's no point sending messages to devices if we couldn't encrypt to them, + // since that's effectively a blank message. + for (const userId of Object.keys(contentMap)) { + for (const deviceId of Object.keys(contentMap[userId])) { + if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) { + logger.log( + "No ciphertext for device " + + userId + ":" + deviceId + ": pruning", + ); + delete contentMap[userId][deviceId]; + } + } + // No devices left for that user? Strip that too. + if (Object.keys(contentMap[userId]).length === 0) { + logger.log("Pruned all devices for user " + userId); + delete contentMap[userId]; + } + } + + // Is there anything left? + if (Object.keys(contentMap).length === 0) { + logger.log("No users left to send to: aborting"); + return; + } + + return this.baseApis.sendToDevice("m.room.encrypted", contentMap); + }); + } + private onToDeviceEvent = (event: MatrixEvent): void => { try { logger.log(`received to_device ${event.getType()} from: ` + From aeeed6ecd761b559aa2b30d1f34c16f3cec210b8 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Mon, 18 Oct 2021 22:44:20 +0100 Subject: [PATCH 100/291] clarify the factoring --- src/crypto/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 94390993ed3..eb1cc8bbaad 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3033,7 +3033,7 @@ export class Crypto extends EventEmitter { /** * @private * Encrypts and sends a given object via Olm to-device message to a given - * set of devices. Heavily derived from encryptAndSendKeysToDevices in + * set of devices. Factored out from encryptAndSendKeysToDevices in * megolm.ts. * * @param {object} userDeviceMap From 24406d24118ce6abe0985b6be73be76000a65f95 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Tue, 19 Oct 2021 00:02:32 +0100 Subject: [PATCH 101/291] make it build --- src/crypto/algorithms/megolm.ts | 27 ++++++++++++++------------- src/crypto/index.ts | 15 ++++++++++----- 2 files changed, 24 insertions(+), 18 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index f771674f5b8..4457c8352b8 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -584,22 +584,23 @@ class MegolmEncryption extends EncryptionAlgorithm { userDeviceMap: IOlmDevice[], payload: IPayload, ): Promise { - this.crypto.encryptAndSendToDevices( + return this.crypto.encryptAndSendToDevices( userDeviceMap, - payload - ).then(() => { - // store that we successfully uploaded the keys of the current slice - for (const userId of Object.keys(contentMap)) { - for (const deviceId of Object.keys(contentMap[userId])) { - session.markSharedWithDevice( - userId, - deviceId, - deviceInfoByDeviceId.get(deviceId).getIdentityKey(), - chainIndex, - ); + payload, + (contentMap, deviceInfoByDeviceId) => { + // store that we successfully uploaded the keys of the current slice + for (const userId of Object.keys(contentMap)) { + for (const deviceId of Object.keys(contentMap[userId])) { + session.markSharedWithDevice( + userId, + deviceId, + deviceInfoByDeviceId.get(deviceId).getIdentityKey(), + chainIndex, + ); + } } } - }); + ); } /** diff --git a/src/crypto/index.ts b/src/crypto/index.ts index eb1cc8bbaad..c1368c3efe5 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -27,6 +27,7 @@ import { EventEmitter } from 'events'; import { ReEmitter } from '../ReEmitter'; import { logger } from '../logger'; import { IExportedDevice, OlmDevice } from "./OlmDevice"; +import { IOlmDevice } from "./algorithms/megolm"; import * as olmlib from "./olmlib"; import { DeviceInfoMap, DeviceList } from "./DeviceList"; import { DeviceInfo, IDevice } from "./deviceinfo"; @@ -3031,7 +3032,6 @@ export class Crypto extends EventEmitter { } /** - * @private * Encrypts and sends a given object via Olm to-device message to a given * set of devices. Factored out from encryptAndSendKeysToDevices in * megolm.ts. @@ -3040,14 +3040,17 @@ export class Crypto extends EventEmitter { * mapping from userId to deviceInfo * * @param {object} payload fields to include in the encrypted payload + * + * @param function to call on each message after it's successfully sent * * @return {Promise} Promise which resolves once the key sharing * for the given userDeviceMap is generated and has been sent. */ - private encryptAndSendToDevices( - userDeviceMap: IExportedDevice[], + encryptAndSendToDevices( + userDeviceMap: IOlmDevice[], payload: object, - ): Promise<{}> { + afterwards: (contentMap: {}, deviceInfoByDeviceId: Map) => void, + ): Promise { const contentMap = {}; const deviceInfoByDeviceId = new Map(); @@ -3110,7 +3113,9 @@ export class Crypto extends EventEmitter { return; } - return this.baseApis.sendToDevice("m.room.encrypted", contentMap); + return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then( + afterwards.bind(contentMap, deviceInfoByDeviceId) + ); }); } From 56c0c9be4d18a6b3ceae64f762fc252ef3c4c30a Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 00:34:09 +0100 Subject: [PATCH 102/291] fix example in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd645d07083..4f41a5a1b2b 100644 --- a/README.md +++ b/README.md @@ -307,7 +307,7 @@ The SDK supports end-to-end encryption via the Olm and Megolm protocols, using [libolm](https://gitlab.matrix.org/matrix-org/olm). It is left up to the application to make libolm available, via the ``Olm`` global. -It is also necessary to call ``matrixClient.initCrypto()`` after creating a new +It is also necessary to call ``await matrixClient.initCrypto()`` after creating a new ``MatrixClient`` (but **before** calling ``matrixClient.startClient()``) to initialise the crypto layer. From 5a83635ef55307364ca4b8624f0500cd1a27695c Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 02:19:32 +0100 Subject: [PATCH 103/291] switch encryptAndSendToDevices to return a promise rather than use a cb and assert that olm sessions are open to the destination devices --- src/crypto/algorithms/megolm.ts | 33 ++++++++++++++++++++------------- src/crypto/index.ts | 23 +++++++++++++++-------- 2 files changed, 35 insertions(+), 21 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index 4457c8352b8..1fdda7d0d76 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -586,21 +586,21 @@ class MegolmEncryption extends EncryptionAlgorithm { ): Promise { return this.crypto.encryptAndSendToDevices( userDeviceMap, - payload, - (contentMap, deviceInfoByDeviceId) => { - // store that we successfully uploaded the keys of the current slice - for (const userId of Object.keys(contentMap)) { - for (const deviceId of Object.keys(contentMap[userId])) { - session.markSharedWithDevice( - userId, - deviceId, - deviceInfoByDeviceId.get(deviceId).getIdentityKey(), - chainIndex, - ); - } + payload + ).then((result) => { + const {contentMap, deviceInfoByDeviceId} = result; + // store that we successfully uploaded the keys of the current slice + for (const userId of Object.keys(contentMap)) { + for (const deviceId of Object.keys(contentMap[userId])) { + session.markSharedWithDevice( + userId, + deviceId, + deviceInfoByDeviceId.get(deviceId).getIdentityKey(), + chainIndex, + ); } } - ); + }); } /** @@ -1496,6 +1496,9 @@ class MegolmDecryption extends DecryptionAlgorithm { return; } } + + // XXX: switch this to use encryptAndSendToDevices() rather than duplicating it? + await olmlib.ensureOlmSessionsForDevices( this.olmDevice, this.baseApis, { [sender]: [device] }, false, ); @@ -1553,6 +1556,8 @@ class MegolmDecryption extends DecryptionAlgorithm { const deviceInfo = this.crypto.getStoredDevice(userId, deviceId); const body = keyRequest.requestBody; + // XXX: switch this to use encryptAndSendToDevices()? + this.olmlib.ensureOlmSessionsForDevices( this.olmDevice, this.baseApis, { [userId]: [deviceInfo], @@ -1735,6 +1740,8 @@ class MegolmDecryption extends DecryptionAlgorithm { for (const [senderKey, sessionId] of sharedHistorySessions) { const payload = await this.buildKeyForwardingMessage(this.roomId, senderKey, sessionId); + // FIXME: use encryptAndSendToDevices() rather than duplicating it here. + const promises = []; const contentMap = {}; for (const [userId, devices] of Object.entries(devicesByUser)) { diff --git a/src/crypto/index.ts b/src/crypto/index.ts index c1368c3efe5..008f975f9a2 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3040,17 +3040,16 @@ export class Crypto extends EventEmitter { * mapping from userId to deviceInfo * * @param {object} payload fields to include in the encrypted payload - * - * @param function to call on each message after it's successfully sent - * - * @return {Promise} Promise which resolves once the key sharing - * for the given userDeviceMap is generated and has been sent. + * * + * @return {Promise<{contentMap, deviceInfoByDeviceId}>} Promise which + * resolves once the message has been encrypted and sent to the given + * userDeviceMap, and returns the { contentMap, deviceInfoByDeviceId } + * of the successfully sent messages. */ encryptAndSendToDevices( userDeviceMap: IOlmDevice[], payload: object, - afterwards: (contentMap: {}, deviceInfoByDeviceId: Map) => void, - ): Promise { + ): Promise<{contentMap, deviceInfoByDeviceId}> { const contentMap = {}; const deviceInfoByDeviceId = new Map(); @@ -3072,7 +3071,15 @@ export class Crypto extends EventEmitter { } contentMap[userId][deviceId] = encryptedContent; + const devicesByUser = {}; + devicesByUser[userId] = [deviceInfo]; + promises.push( + olmlib.ensureOlmSessionsForDevices( + this.olmDevice, + this.baseApis, + devicesByUser, + ), olmlib.encryptMessageForDevice( encryptedContent.ciphertext, this.userId, @@ -3114,7 +3121,7 @@ export class Crypto extends EventEmitter { } return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then( - afterwards.bind(contentMap, deviceInfoByDeviceId) + (response)=>({ contentMap, deviceInfoByDeviceId }) ); }); } From 53397ee0d17e76d04586bf9c9d00f636f458c733 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 02:21:12 +0100 Subject: [PATCH 104/291] lint --- src/crypto/algorithms/megolm.ts | 4 ++-- src/crypto/index.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index 1fdda7d0d76..c94da5980c5 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -586,9 +586,9 @@ class MegolmEncryption extends EncryptionAlgorithm { ): Promise { return this.crypto.encryptAndSendToDevices( userDeviceMap, - payload + payload, ).then((result) => { - const {contentMap, deviceInfoByDeviceId} = result; + const { contentMap, deviceInfoByDeviceId } = result; // store that we successfully uploaded the keys of the current slice for (const userId of Object.keys(contentMap)) { for (const deviceId of Object.keys(contentMap[userId])) { diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 008f975f9a2..14d738d09b2 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3046,7 +3046,7 @@ export class Crypto extends EventEmitter { * userDeviceMap, and returns the { contentMap, deviceInfoByDeviceId } * of the successfully sent messages. */ - encryptAndSendToDevices( + public encryptAndSendToDevices( userDeviceMap: IOlmDevice[], payload: object, ): Promise<{contentMap, deviceInfoByDeviceId}> { @@ -3121,7 +3121,7 @@ export class Crypto extends EventEmitter { } return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then( - (response)=>({ contentMap, deviceInfoByDeviceId }) + (response)=>({ contentMap, deviceInfoByDeviceId }), ); }); } From f46355e7c0415b538bb9a908189fe4de91f669eb Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 03:16:18 +0100 Subject: [PATCH 105/291] don't choke on missing promise --- src/crypto/algorithms/megolm.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index c94da5980c5..36ae066e704 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -584,10 +584,13 @@ class MegolmEncryption extends EncryptionAlgorithm { userDeviceMap: IOlmDevice[], payload: IPayload, ): Promise { - return this.crypto.encryptAndSendToDevices( + const p = this.crypto.encryptAndSendToDevices( userDeviceMap, payload, - ).then((result) => { + ); + if (!p) return; + return p.then((result) => { + if (!result) return; const { contentMap, deviceInfoByDeviceId } = result; // store that we successfully uploaded the keys of the current slice for (const userId of Object.keys(contentMap)) { From 2f09e9641cca845d9a21e84bd7e72ff756294a22 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 05:08:03 +0100 Subject: [PATCH 106/291] chain promises correctly; log rejects --- src/crypto/index.ts | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 14d738d09b2..ebd4e168774 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3079,16 +3079,17 @@ export class Crypto extends EventEmitter { this.olmDevice, this.baseApis, devicesByUser, - ), - olmlib.encryptMessageForDevice( - encryptedContent.ciphertext, - this.userId, - this.deviceId, - this.olmDevice, - userId, - deviceInfo, - payload, - ), + ).then(()=> + olmlib.encryptMessageForDevice( + encryptedContent.ciphertext, + this.userId, + this.deviceId, + this.olmDevice, + userId, + deviceInfo, + payload, + ) + ) ); } @@ -3122,7 +3123,13 @@ export class Crypto extends EventEmitter { return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then( (response)=>({ contentMap, deviceInfoByDeviceId }), - ); + ).catch(error=>{ + console.error("sendToDevice failed", error); + throw error; + }); + }).catch(error=>{ + console.error("encryptAndSendToDevices promises failed", error); + throw error; }); } From a48546f60d7d4a9a41470ed3e3b925a34220ff32 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 05:09:08 +0100 Subject: [PATCH 107/291] fix the tests (thanks @turt2live!!!) --- spec/unit/crypto/algorithms/megolm.spec.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/unit/crypto/algorithms/megolm.spec.js b/spec/unit/crypto/algorithms/megolm.spec.js index d949b1bed58..73d98d9d081 100644 --- a/spec/unit/crypto/algorithms/megolm.spec.js +++ b/spec/unit/crypto/algorithms/megolm.spec.js @@ -321,6 +321,12 @@ describe("MegolmDecryption", function() { rotation_period_ms: 9999999999999, }, }); + + // Fix the mock to call the stuff we need it to + mockCrypto.encryptAndSendToDevices = Crypto.prototype.encryptAndSendToDevices; + mockCrypto.olmDevice = olmDevice; + mockCrypto.baseApis = mockBaseApis; + mockRoom = { getEncryptionTargetMembers: jest.fn().mockReturnValue( [{ userId: "@alice:home.server" }], From 841e6e999df5c66c5d6703800a6560e2080672f6 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 05:09:28 +0100 Subject: [PATCH 108/291] handle promises normally now tests are fixed --- src/crypto/algorithms/megolm.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index 36ae066e704..ba03c8b9ade 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -584,13 +584,10 @@ class MegolmEncryption extends EncryptionAlgorithm { userDeviceMap: IOlmDevice[], payload: IPayload, ): Promise { - const p = this.crypto.encryptAndSendToDevices( + return this.crypto.encryptAndSendToDevices( userDeviceMap, payload, - ); - if (!p) return; - return p.then((result) => { - if (!result) return; + ).then((result) => { const { contentMap, deviceInfoByDeviceId } = result; // store that we successfully uploaded the keys of the current slice for (const userId of Object.keys(contentMap)) { @@ -603,6 +600,9 @@ class MegolmEncryption extends EncryptionAlgorithm { ); } } + }).catch((error) => { + console.error("failed to encryptAndSendToDevices", error); + throw error; }); } From 067ac622711171ebe9fcac8a6d8a7ec35b6d8cb4 Mon Sep 17 00:00:00 2001 From: Matthew Hodgson Date: Wed, 20 Oct 2021 05:16:39 +0100 Subject: [PATCH 109/291] lint --- src/crypto/algorithms/megolm.ts | 2 +- src/crypto/index.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index ba03c8b9ade..d1f462d7031 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -601,7 +601,7 @@ class MegolmEncryption extends EncryptionAlgorithm { } } }).catch((error) => { - console.error("failed to encryptAndSendToDevices", error); + logger.error("failed to encryptAndSendToDevices", error); throw error; }); } diff --git a/src/crypto/index.ts b/src/crypto/index.ts index ebd4e168774..e44184b9ced 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3088,8 +3088,8 @@ export class Crypto extends EventEmitter { userId, deviceInfo, payload, - ) - ) + ), + ), ); } @@ -3124,11 +3124,11 @@ export class Crypto extends EventEmitter { return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then( (response)=>({ contentMap, deviceInfoByDeviceId }), ).catch(error=>{ - console.error("sendToDevice failed", error); + logger.error("sendToDevice failed", error); throw error; }); }).catch(error=>{ - console.error("encryptAndSendToDevices promises failed", error); + logger.error("encryptAndSendToDevices promises failed", error); throw error; }); } From 13d62e71b6f5130f3cbeede6dbeb5df2f52212f8 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 26 Oct 2021 16:50:56 -0700 Subject: [PATCH 110/291] Fix stopping all media streams --- src/webrtc/mediaHandler.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 597c766b832..2c34b926d9c 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -201,6 +201,7 @@ export class MediaHandler { this.userMediaStreams = []; this.screensharingStreams = []; + this.localUserMediaStream = undefined; } private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints { From 923e9c4adacfd0b781dbe3027468dad44f7a5d50 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 28 Oct 2021 12:25:00 -0700 Subject: [PATCH 111/291] Ensure that member call state is set correctly --- src/webrtc/groupCall.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d38db94847f..c2fe5985f4a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -551,11 +551,13 @@ export class GroupCall extends EventEmitter { existingCallIndex = -1; } - if (existingCallIndex === -1) { - calls.push(memberCallState); - } else if (memberCallState) { - calls.splice(existingCallIndex, 1, memberCallState); - } else { + if (memberCallState) { + if (existingCallIndex !== -1) { + calls.splice(existingCallIndex, 1, memberCallState); + } else { + calls.push(memberCallState); + } + } else if (existingCallIndex !== -1) { calls.splice(existingCallIndex, 1); } @@ -582,14 +584,12 @@ export class GroupCall extends EventEmitter { const callsState = event.getContent()["m.calls"]; - if (!callsState || !Array.isArray(callsState) || callsState.length === 0) { + if (!callsState || !Array.isArray(callsState) || callsState.length === 0 || !callsState[0]) { logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); this.removeParticipant(member); return; } - logger.log(callsState); - // Currently we only support a single call per room. So grab the first call. const callState = callsState[0]; From 8c5f88c4a770411feca96a96f5bf3e17c155835c Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 28 Oct 2021 13:27:35 -0700 Subject: [PATCH 112/291] Fix handling null call --- src/webrtc/groupCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c2fe5985f4a..9efe0ef87a3 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -660,7 +660,8 @@ export class GroupCall extends EventEmitter { } const memberState = memberStateEvent.getContent(); - const memberGroupCallState = memberState["m.calls"]?.find((call) => call["m.call_id"] === this.groupCallId); + const memberGroupCallState = memberState["m.calls"]?.find( + (call) => call && call["m.call_id"] === this.groupCallId); if (!memberGroupCallState) { return undefined; From 4c9648a23b219da540886bd185abc4783283f4b4 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 28 Oct 2021 13:46:27 -0700 Subject: [PATCH 113/291] Sanitize call member state --- src/webrtc/groupCall.ts | 30 +++++++++++++++++------------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 9efe0ef87a3..5a475669d48 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -543,22 +543,22 @@ export class GroupCall extends EventEmitter { const memberStateEvent = currentStateEvent?.getContent(); let calls: IGroupCallRoomMemberCallState[] = []; - let existingCallIndex: number; - if (memberStateEvent) { - calls = memberStateEvent["m.calls"] || []; - existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); - } else { - existingCallIndex = -1; + + // Sanitize existing member state event + if (memberStateEvent && Array.isArray(memberStateEvent["m.calls"])) { + calls = memberStateEvent["m.calls"].filter((call) => !!call); } - if (memberCallState) { - if (existingCallIndex !== -1) { + const existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); + + if (existingCallIndex !== -1) { + if (memberCallState) { calls.splice(existingCallIndex, 1, memberCallState); } else { - calls.push(memberCallState); + calls.splice(existingCallIndex, 1); } - } else if (existingCallIndex !== -1) { - calls.splice(existingCallIndex, 1); + } else if (memberCallState) { + calls.push(memberCallState); } const content = { @@ -582,9 +582,13 @@ export class GroupCall extends EventEmitter { return; } - const callsState = event.getContent()["m.calls"]; + let callsState = event.getContent()["m.calls"]; + + if (Array.isArray(callsState)) { + callsState = callsState.filter((call) => !!call); + } - if (!callsState || !Array.isArray(callsState) || callsState.length === 0 || !callsState[0]) { + if (!Array.isArray(callsState) || callsState.length === 0) { logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); this.removeParticipant(member); return; From d99428f2c1fc6254aef7685e6d7e1296e358c7b0 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 2 Nov 2021 11:39:45 -0700 Subject: [PATCH 114/291] Remove duplicate call answer --- src/webrtc/callEventHandler.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 3bf66e47fae..b9c281b9a3d 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -273,7 +273,6 @@ export class CallEventHandler { " and canceling outgoing call " + existingCall.callId, ); existingCall.replacedBy(call); - call.answer(); } else { logger.log( "Glare detected: rejecting incoming call " + call.callId + From 6f695c1b82a5575d4897558b422e28a122d9c05c Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 2 Nov 2021 15:30:38 -0700 Subject: [PATCH 115/291] Ignore call call state in glare resolution --- src/webrtc/callEventHandler.ts | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index b9c281b9a3d..8574d0c3561 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -259,15 +259,7 @@ export class CallEventHandler { } if (existingCall) { - // If we've only got to wait_local_media or create_offer and - // we've got an invite, pick the incoming call because we know - // we haven't sent our invite yet otherwise, pick whichever - // call has the lowest call ID (by string comparison) - if ( - existingCall.state === CallState.WaitLocalMedia || - existingCall.state === CallState.CreateOffer || - existingCall.callId > call.callId - ) { + if (existingCall.callId > call.callId) { logger.log( "Glare detected: answering incoming call " + call.callId + " and canceling outgoing call " + existingCall.callId, From 5a92597abdd3b750003f6e9a23e487ef8bea8ca1 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 2 Nov 2021 15:33:58 -0700 Subject: [PATCH 116/291] Check if call ended before getting user media --- src/webrtc/call.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7b883b74e06..645664b0244 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1359,6 +1359,11 @@ export class MatrixCall extends EventEmitter { this.getRidOfRTXCodecs(); await this.peerConn.setLocalDescription(); this.setState(CallState.Connecting); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } // Allow a short time for initial candidates to be gathered await new Promise(resolve => { @@ -1376,12 +1381,11 @@ export class MatrixCall extends EventEmitter { setTimeout(resolve, 200); }); - await this.client.getMediaHandler().updateLocalUsermediaStreams(); + if (this.state === CallState.Ended) { + return; } - } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); - return; + + await this.client.getMediaHandler().updateLocalUsermediaStreams(); } } From bdc12a2544080fbc283c9189498ce51260e51ef3 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 2 Nov 2021 17:25:41 -0700 Subject: [PATCH 117/291] Revert changes to gotCallFeedsForAnswer --- src/webrtc/call.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 645664b0244..db7760e605e 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1355,22 +1355,31 @@ export class MatrixCall extends EventEmitter { this.setState(CallState.CreateAnswer); + let myAnswer; try { this.getRidOfRTXCodecs(); - await this.peerConn.setLocalDescription(); - this.setState(CallState.Connecting); + myAnswer = await this.peerConn.createAnswer(); } catch (err) { - logger.debug("Error setting local description!", err); - this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + logger.debug("Failed to create answer: ", err); + this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); return; } + try { + await this.peerConn.setLocalDescription(myAnswer); + this.setState(CallState.Connecting); + // Allow a short time for initial candidates to be gathered await new Promise(resolve => { setTimeout(resolve, 200); }); this.sendAnswer(); + } catch (err) { + logger.debug("Error setting local description!", err); + this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); + return; + } // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases // we can get around this by calling MediaStream.clone(), however inbound From 7998817f7e8b3732f365fcaf4aa668dccf15ba5b Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 4 Nov 2021 11:44:11 -0700 Subject: [PATCH 118/291] Send candidate queue again on finish to flush out queue --- src/webrtc/call.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index db7760e605e..74d2ae731ad 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2125,6 +2125,9 @@ export class MatrixCall extends EventEmitter { // reset our retry count if we have successfully sent our candidates // otherwise queueCandidate() will refuse to try to flush the queue this.candidateSendTries = 0; + + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); } catch (error) { // don't retry this event: we'll send another one later as we might // have more candidates by then. From c1f56ba3c4714c1469381f64b45e2a2963848e79 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 4 Nov 2021 11:46:20 -0700 Subject: [PATCH 119/291] Fix indentation --- src/webrtc/call.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 74d2ae731ad..cbd2067d628 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1381,20 +1381,20 @@ export class MatrixCall extends EventEmitter { return; } - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); if (this.state === CallState.Ended) { return; } - await this.client.getMediaHandler().updateLocalUsermediaStreams(); + await this.client.getMediaHandler().updateLocalUsermediaStreams(); } } From b253ad9e81f8e84e4508ae45d1a9947bc45c68dd Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 4 Nov 2021 17:22:37 -0700 Subject: [PATCH 120/291] Preserve the disabled tracks when updating local usermedia stream --- src/webrtc/call.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cbd2067d628..7795abc775b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1095,6 +1095,10 @@ export class MatrixCall extends EventEmitter { const callFeed = this.localUsermediaFeed; callFeed.setNewStream(stream); + const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; + const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); const newSenders = []; From 17f7dc5463875a95a90fa6d9b4c80824ea75e3bb Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 4 Nov 2021 17:44:47 -0700 Subject: [PATCH 121/291] Keep track of original stream id for sdp stream metadata --- src/webrtc/call.ts | 14 +++++++++----- src/webrtc/callFeed.ts | 2 ++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 7795abc775b..cab612b63ab 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -481,10 +481,14 @@ export class MatrixCall extends EventEmitter { * Generates and returns localSDPStreamMetadata * @returns {SDPStreamMetadata} localSDPStreamMetadata */ - private getLocalSDPStreamMetadata(): SDPStreamMetadata { + private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { const metadata: SDPStreamMetadata = {}; for (const localFeed of this.getLocalFeeds()) { - metadata[localFeed.stream.id] = { + if (updateStreamIds) { + localFeed.sdpMetadataStreamId = localFeed.stream.id; + } + + metadata[localFeed.sdpMetadataStreamId] = { purpose: localFeed.purpose, audio_muted: localFeed.isAudioMuted(), video_muted: localFeed.isVideoMuted(), @@ -1309,7 +1313,7 @@ export class MatrixCall extends EventEmitter { // required to still be sent for backwards compat type: this.peerConn.localDescription.type, }, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), } as MCallAnswer; answerContent.capabilities = { @@ -1599,7 +1603,7 @@ export class MatrixCall extends EventEmitter { this.sendVoipEvent(EventType.CallNegotiate, { description: this.peerConn.localDescription, - [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(true), }); } } catch (err) { @@ -1696,7 +1700,7 @@ export class MatrixCall extends EventEmitter { 'm.call.dtmf': false, }; - content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(); + content[SDPStreamMetadataKey] = this.getLocalSDPStreamMetadata(true); // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 6de031a1074..df754886652 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -42,6 +42,7 @@ export enum CallFeedEvent { export class CallFeed extends EventEmitter { public stream: MediaStream; + public sdpMetadataStreamId: string; public userId: string; public purpose: SDPStreamMetadataPurpose; public speakingVolumeSamples: number[]; @@ -68,6 +69,7 @@ export class CallFeed extends EventEmitter { this.audioMuted = opts.audioMuted; this.videoMuted = opts.videoMuted; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.sdpMetadataStreamId = opts.stream.id; this.updateStream(null, opts.stream); From f76f708c96b8e36d91fbac39d81a63aa221b0bb5 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 5 Nov 2021 14:31:38 -0700 Subject: [PATCH 122/291] Ad a longer wait to safari media stream hack --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cab612b63ab..1595a703477 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1395,7 +1395,7 @@ export class MatrixCall extends EventEmitter { // all MediaStreams using sender.replaceTrack if (isSafari) { await new Promise(resolve => { - setTimeout(resolve, 200); + setTimeout(resolve, 2000); }); if (this.state === CallState.Ended) { From 4b87907b92474463c9aaa49715cbbc4f512a7519 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 8 Nov 2021 12:30:56 -0800 Subject: [PATCH 123/291] Update local usermedia streams serially --- src/webrtc/mediaHandler.ts | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 2c34b926d9c..278a8b9c428 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -54,20 +54,27 @@ export class MediaHandler { public async updateLocalUsermediaStreams(): Promise { this.localUserMediaStream = undefined; - await Promise.all(Array.from(this.client.callEventHandler.calls.values()) - .filter((call) => call.state !== CallState.Ended) - .map((call) => { - return this.getUserMediaStream( - call.hasLocalUserMediaAudioTrack, - call.hasLocalUserMediaVideoTrack, - ).then(stream => call.updateLocalUsermediaStream(stream)); - })); - - await Promise.all(Array.from(this.client.groupCallEventHandler.groupCalls.values()) - .map((groupCall) => { - return this.getUserMediaStream(true, groupCall.type === GroupCallType.Video) - .then(stream => groupCall.updateLocalUsermediaStream(stream)); - })); + for (const call of this.client.callEventHandler.calls.values()) { + if (call.state === CallState.Ended) { + continue; + } + + const stream = await this.getUserMediaStream( + call.hasLocalUserMediaAudioTrack, + call.hasLocalUserMediaVideoTrack, + ); + + await call.updateLocalUsermediaStream(stream); + } + + for (const groupCall of this.client.groupCallEventHandler.groupCalls.values()) { + const stream = await this.getUserMediaStream( + true, + groupCall.type === GroupCallType.Video, + ); + + await groupCall.updateLocalUsermediaStream(stream); + } } public async hasAudioDevice(): Promise { From 74c5a203719dce80d4a765519b6b1e69fab3781c Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 8 Nov 2021 12:57:38 -0800 Subject: [PATCH 124/291] Temporarily disable safari hack --- src/webrtc/call.ts | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index db7760e605e..b5fe7f65eae 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1381,21 +1381,21 @@ export class MatrixCall extends EventEmitter { return; } - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - await this.client.getMediaHandler().updateLocalUsermediaStreams(); - } + // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases + // we can get around this by calling MediaStream.clone(), however inbound + // calls seem to still be broken unless we getUserMedia again and replace + // all MediaStreams using sender.replaceTrack + // if (isSafari) { + // await new Promise(resolve => { + // setTimeout(resolve, 200); + // }); + + // if (this.state === CallState.Ended) { + // return; + // } + + // await this.client.getMediaHandler().updateLocalUsermediaStreams(); + // } } /** From f28cb48fe1ca2cbb34dabe8bff610d4f63413f58 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 8 Nov 2021 13:07:22 -0800 Subject: [PATCH 125/291] Re-enable safari hack --- src/webrtc/call.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 925aa13be2a..cab612b63ab 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -228,7 +228,7 @@ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; -// const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); +const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); export class CallError extends Error { code: string; @@ -1393,17 +1393,17 @@ export class MatrixCall extends EventEmitter { // we can get around this by calling MediaStream.clone(), however inbound // calls seem to still be broken unless we getUserMedia again and replace // all MediaStreams using sender.replaceTrack - // if (isSafari) { - // await new Promise(resolve => { - // setTimeout(resolve, 200); - // }); + if (isSafari) { + await new Promise(resolve => { + setTimeout(resolve, 200); + }); - // if (this.state === CallState.Ended) { - // return; - // } + if (this.state === CallState.Ended) { + return; + } - // await this.client.getMediaHandler().updateLocalUsermediaStreams(); - // } + await this.client.getMediaHandler().updateLocalUsermediaStreams(); + } } /** From 97e8fcea758e0205bd9a8e516550d95412e364ee Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 9 Nov 2021 14:01:16 -0800 Subject: [PATCH 126/291] Clean up replacing calls for Safari --- src/webrtc/call.ts | 60 ++++++++++++++++++++++++++++++++++++----- src/webrtc/groupCall.ts | 4 +++ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cab612b63ab..d7f198c307d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -886,7 +886,7 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { logger.debug("Handing local stream to new call"); - newCall.gotCallFeedsForAnswer(this.getLocalFeeds().map((feed) => feed.clone())); + newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -1402,7 +1402,54 @@ export class MatrixCall extends EventEmitter { return; } - await this.client.getMediaHandler().updateLocalUsermediaStreams(); + const callFeed = this.localUsermediaFeed; + const stream = callFeed.stream; + + if (!stream.active) { + throw new Error(`Call ${this.callId} has an inactive stream ${ + stream.id} and its tracks cannot be replaced`); + } + + const newSenders = []; + + for (const track of this.localUsermediaStream.getTracks()) { + const oldSender = this.usermediaSenders.find((sender) => { + return sender.track?.kind === track.kind; + }); + + if (track.readyState === "ended") { + throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); + } + + let newSender: RTCRtpSender; + + try { + logger.info( + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + newSender = oldSender; + } catch (error) { + logger.info( + `Adding track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + newSender = this.peerConn.addTrack(track, stream); + } + + newSenders.push(newSender); + } + + this.usermediaSenders = newSenders; } } @@ -2058,7 +2105,9 @@ export class MatrixCall extends EventEmitter { private async terminate(hangupParty: CallParty, hangupReason: CallErrorCode, shouldEmit: boolean): Promise { if (this.callHasEnded()) return; - this.callStatsAtEnd = await this.collectCallStats(); + this.hangupParty = hangupParty; + this.hangupReason = hangupReason; + this.setState(CallState.Ended); if (this.inviteTimeout) { clearTimeout(this.inviteTimeout); @@ -2069,14 +2118,13 @@ export class MatrixCall extends EventEmitter { this.callLengthInterval = null; } + this.callStatsAtEnd = await this.collectCallStats(); + // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() // We don't stop media if the call was replaced as we want to re-use streams in the successor if (hangupReason !== CallErrorCode.Replaced) this.stopAllMedia(); this.deleteAllFeeds(); - this.hangupParty = hangupParty; - this.hangupReason = hangupReason; - this.setState(CallState.Ended); if (this.peerConn && this.peerConn.signalingState !== 'closed') { this.peerConn.close(); } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 5a475669d48..b20126f4aba 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -775,6 +775,10 @@ export class GroupCall extends EventEmitter { this.callHandlers.delete(opponentMemberId); + if (call.hangupReason === CallErrorCode.Replaced) { + return; + } + if (call.state !== CallState.Ended) { call.hangup(hangupReason, false); } From e880cece93a7fb830e127832c461c1973e086c45 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 9 Nov 2021 14:40:29 -0800 Subject: [PATCH 127/291] Add restart ICE --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d7f198c307d..d1f6770bd81 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1844,7 +1844,7 @@ export class MatrixCall extends EventEmitter { }, 1000); } } else if (this.peerConn.iceConnectionState == 'failed') { - this.hangup(CallErrorCode.IceFailed, false); + this.peerConn.restartIce(); } else if (this.peerConn.iceConnectionState == 'disconnected') { this.iceDisconnectedTimeout = setTimeout(() => { this.hangup(CallErrorCode.IceFailed, false); From 28da62c01c68bab94e3d778237abca8ed699c4e6 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 9 Nov 2021 15:31:27 -0800 Subject: [PATCH 128/291] Add retry call loop --- src/webrtc/groupCall.ts | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b20126f4aba..a5ec63c649d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -104,6 +104,7 @@ function getCallUserId(call: MatrixCall): string | null { export class GroupCall extends EventEmitter { // Config public activeSpeakerInterval = 1000; + public retryCallInterval = 5000; public participantTimeout = 1000 * 15; public state = GroupCallState.LocalCallFeedUninitialized; @@ -119,6 +120,8 @@ export class GroupCall extends EventEmitter { private callHandlers: Map = new Map(); private activeSpeakerLoopTimeout?: number; + private retryCallLoopTimeout?: number; + private retryCallCounts: Map = new Map(); private reEmitter: ReEmitter; constructor( @@ -274,6 +277,7 @@ export class GroupCall extends EventEmitter { this.client.on("Call.incoming", this.onIncomingCall); this.onActiveSpeakerLoop(); + this.onRetryCallLoop(); } private dispose() { @@ -305,6 +309,10 @@ export class GroupCall extends EventEmitter { this.activeSpeaker = null; clearTimeout(this.activeSpeakerLoopTimeout); + + this.retryCallCounts.clear(); + clearTimeout(this.retryCallLoopTimeout); + this.client.removeListener("Call.incoming", this.onIncomingCall); } @@ -681,6 +689,24 @@ export class GroupCall extends EventEmitter { return memberDevices[0]; } + private onRetryCallLoop = () => { + const roomState = this.room.currentState; + const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); + + for (const event of memberStateEvents) { + const memberId = event.getStateKey(); + const existingCall = this.calls.find((call) => getCallUserId(call) === memberId); + const retryCallCount = this.retryCallCounts.get(memberId) || 0; + + if (!existingCall && retryCallCount < 3) { + this.retryCallCounts.set(memberId, retryCallCount + 1); + this.onMemberStateChanged(event); + } + } + + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); + }; + /** * Call Event Handlers */ @@ -832,7 +858,7 @@ export class GroupCall extends EventEmitter { } }; - private onCallStateChanged = (call: MatrixCall, _state: CallState, _oldState: CallState) => { + private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState) => { const audioMuted = this.localCallFeed.isAudioMuted(); if ( @@ -850,6 +876,10 @@ export class GroupCall extends EventEmitter { ) { call.setLocalVideoMuted(videoMuted); } + + if (state === CallState.Connected) { + this.retryCallCounts.delete(getCallUserId(call)); + } }; private onCallHangup = (call: MatrixCall) => { From 7d48a8394d785be5f86d7cc0966f8d92000d138e Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 15 Nov 2021 10:48:24 -0800 Subject: [PATCH 129/291] Don't immediately start retry call loop --- src/webrtc/groupCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a5ec63c649d..37f4708026c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -277,7 +277,8 @@ export class GroupCall extends EventEmitter { this.client.on("Call.incoming", this.onIncomingCall); this.onActiveSpeakerLoop(); - this.onRetryCallLoop(); + + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); } private dispose() { From 31391121dcc6a30acfb3214ec0c4f8cbd24d00ca Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 15 Nov 2021 11:38:47 -0800 Subject: [PATCH 130/291] Clean up logging --- src/webrtc/call.ts | 4 ---- src/webrtc/groupCall.ts | 21 ++++----------------- src/webrtc/groupCallEventHandler.ts | 6 +++--- 3 files changed, 7 insertions(+), 24 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d1f6770bd81..9399166d019 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -365,7 +365,6 @@ export class MatrixCall extends EventEmitter { public createDataChannel(label: string, options: RTCDataChannelInit) { const dataChannel = this.peerConn.createDataChannel(label, options); this.emit(CallEvent.DataChannel, dataChannel); - logger.debug("created data channel"); return dataChannel; } @@ -494,7 +493,6 @@ export class MatrixCall extends EventEmitter { video_muted: localFeed.isVideoMuted(), }; } - logger.debug("Got local SDPStreamMetadata", metadata); return metadata; } @@ -679,8 +677,6 @@ export class MatrixCall extends EventEmitter { } private deleteFeedByStream(stream: MediaStream): void { - logger.debug(`Removing feed with stream id ${stream.id}`); - const feed = this.getFeedByStreamId(stream.id); if (!feed) { logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 37f4708026c..e7812dddd4d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -140,8 +140,6 @@ export class GroupCall extends EventEmitter { const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); - logger.log("Processing initial members", memberStateEvents); - for (const stateEvent of memberStateEvents) { this.onMemberStateChanged(stateEvent); } @@ -250,8 +248,6 @@ export class GroupCall extends EventEmitter { logger.log(`Entered group call ${this.groupCallId}`); - logger.log("processing initial calls"); - const calls = this.client.callEventHandler.calls.values(); for (const call of calls) { @@ -263,8 +259,6 @@ export class GroupCall extends EventEmitter { const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); - logger.log("Processing initial members"); - // This avoids a race condition where the other side would first receive // the to-device messages and only then the member state event which // would result in the call being ignored @@ -446,11 +440,9 @@ export class GroupCall extends EventEmitter { await this.sendMemberStateEvent(); - logger.log("screensharing enabled on all calls"); - return true; } catch (error) { - logger.error("enabling screensharing error", error); + logger.error("Enabling screensharing error", error); this.emit(GroupCallEvent.Error, new GroupCallError(GroupCallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", error), ); @@ -573,7 +565,7 @@ export class GroupCall extends EventEmitter { const content = { "m.calls": calls, }; - logger.log("Sending group call member state event", content); + return this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId); } @@ -585,8 +577,6 @@ export class GroupCall extends EventEmitter { const member = this.room.getMember(event.getStateKey()); - logger.log("Processing member state", member); - if (!member) { return; } @@ -598,7 +588,7 @@ export class GroupCall extends EventEmitter { } if (!Array.isArray(callsState) || callsState.length === 0) { - logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); + logger.warn(`Ignoring member state from ${member.userId} member not in any calls.`); this.removeParticipant(member); return; } @@ -615,7 +605,7 @@ export class GroupCall extends EventEmitter { } if (callId !== this.groupCallId) { - logger.log(`Call id ${callId} does not match group call id ${this.groupCallId}, ignoring.`); + logger.warn(`Call id ${callId} does not match group call id ${this.groupCallId}, ignoring.`); this.removeParticipant(member); return; } @@ -984,13 +974,11 @@ export class GroupCall extends EventEmitter { } private addScreenshareFeed(callFeed: CallFeed) { - logger.log("added screenshare feed"); this.screenshareFeeds.push(callFeed); this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); } private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { - logger.log("replaced screenshare feed"); const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === existingFeed.userId); if (feedIndex === -1) { @@ -1004,7 +992,6 @@ export class GroupCall extends EventEmitter { } private removeScreenshareFeed(callFeed: CallFeed) { - logger.log("removed screenshare feed"); const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === callFeed.userId); if (feedIndex === -1) { diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index c8d7485b15d..99e726f1f1c 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -73,7 +73,7 @@ export class GroupCallEventHandler { const room = this.client.getRoom(roomId); if (!room) { - logger.error(`Couldn't find room ${roomId} for GroupCall`); + logger.warn(`Couldn't find room ${roomId} for GroupCall`); return; } @@ -82,14 +82,14 @@ export class GroupCallEventHandler { const callType = content["m.type"]; if (!Object.values(GroupCallType).includes(callType)) { - logger.error(`Received invalid group call type ${callType} for room ${roomId}.`); + logger.warn(`Received invalid group call type ${callType} for room ${roomId}.`); return; } const callIntent = content["m.intent"]; if (!Object.values(GroupCallIntent).includes(callIntent)) { - logger.error(`Received invalid group call intent ${callType} for room ${roomId}.`); + logger.warn(`Received invalid group call intent ${callType} for room ${roomId}.`); return; } From 4e26f29032c001b7bb8d1ae9dd5ae6a81aac1ae8 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 15 Nov 2021 12:05:34 -0800 Subject: [PATCH 131/291] Add unknown device errors --- src/webrtc/callEventHandler.ts | 8 ++++++++ src/webrtc/groupCall.ts | 18 +++++++++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 8574d0c3561..fef13faa924 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -21,6 +21,7 @@ import { EventType } from '../@types/event'; import { MatrixClient } from '../client'; import { MCallAnswer, MCallHangupReject } from "./callEventTypes"; import { SyncState } from "../sync.api"; +import { GroupCallError, GroupCallErrorCode, GroupCallEvent } from './groupCall'; // Don't ring unless we'd be ringing for at least 3 seconds: the user needs some // time to press the 'accept' button @@ -186,6 +187,13 @@ export class CallEventHandler { if (!opponentDeviceId) { logger.warn(`Cannot find a device id for ${senderId}. Ignoring event.`); + groupCall.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.UnknownDevice, + `No opponent device found for ${senderId}, ignoring.`, + ), + ); return; } } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e7812dddd4d..92549b6d0a1 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -40,15 +40,20 @@ export enum GroupCallEvent { } export enum GroupCallErrorCode { - NoUserMedia = "no_user_media" + NoUserMedia = "no_user_media", + UnknownDevice = "unknown_device" } export class GroupCallError extends Error { code: string; - constructor(code: GroupCallErrorCode, msg: string, err: Error) { + constructor(code: GroupCallErrorCode, msg: string, err?: Error) { // Still don't think there's any way to have proper nested errors - super(msg + ": " + err); + if (err) { + super(msg + ": " + err); + } else { + super(msg); + } this.code = code; } @@ -633,6 +638,13 @@ export class GroupCall extends EventEmitter { if (!opponentDevice) { logger.warn(`No opponent device found for ${member.userId}, ignoring.`); + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.UnknownDevice, + `No opponent device found for ${member.userId}, ignoring.`, + ), + ); return; } From c2fe2ab2707cd874a2a250efde24eaa0b542d310 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 15 Nov 2021 14:20:55 -0800 Subject: [PATCH 132/291] Add additional logging for removing feeds/tracks --- src/webrtc/call.ts | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9399166d019..baa784cdd24 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -662,7 +662,7 @@ export class MatrixCall extends EventEmitter { } // Empty the array senderArray.splice(0, senderArray.length); - this.deleteFeedByStream(callFeed.stream); + this.deleteFeed(callFeed); } private deleteAllFeeds(): void { @@ -682,7 +682,10 @@ export class MatrixCall extends EventEmitter { logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); return; } + this.deleteFeed(feed); + } + private deleteFeed(feed: CallFeed): void { feed.dispose(); this.feeds.splice(this.feeds.indexOf(feed), 1); this.emit(CallEvent.FeedsChanged, this.feeds); @@ -1863,7 +1866,10 @@ export class MatrixCall extends EventEmitter { const stream = ev.streams[0]; this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => this.deleteFeedByStream(stream)); + stream.addEventListener("removetrack", () => { + logger.log(`Removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + }); }; private onDataChannel = (ev: RTCDataChannelEvent): void => { From 139904f29701758849b214c205a76223e9359094 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 17 Nov 2021 16:05:02 -0800 Subject: [PATCH 133/291] Update sync state to error when aborting --- src/sync.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sync.ts b/src/sync.ts index 52bbbefc5b8..2a0030bffc0 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -505,6 +505,7 @@ export class SyncApi { // The logout already happened, we just need to stop. logger.warn("Token no longer valid - assuming logout"); this.stop(); + this.updateSyncState(SyncState.Error, { error }); return true; } return false; From 5f06df8a878646bd39c023b4258b067000727339 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 18 Nov 2021 13:53:30 -0800 Subject: [PATCH 134/291] Properly stop screensharing feed --- src/webrtc/call.ts | 5 +++++ src/webrtc/groupCall.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 93e04529cf9..33afdfbcb34 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -660,6 +660,11 @@ export class MatrixCall extends EventEmitter { for (const sender of senderArray) { this.peerConn.removeTrack(sender); } + + if (callFeed.purpose === SDPStreamMetadataPurpose.Screenshare) { + this.client.getMediaHandler().stopScreensharingStream(callFeed.stream); + } + // Empty the array senderArray.splice(0, senderArray.length); this.deleteFeed(callFeed); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 92549b6d0a1..bb79d0441a2 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -454,7 +454,7 @@ export class GroupCall extends EventEmitter { return false; } } else { - await Promise.all(this.calls.map(call => call.removeLocalFeed(this.localScreenshareFeed))); + await Promise.all(this.calls.map(call => call.removeLocalFeed(call.localScreensharingFeed))); this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; From 64c3ac55a43eaa5f0cf7bec2324e05930b09a41f Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 19 Nov 2021 13:56:50 -0800 Subject: [PATCH 135/291] Stop screenshare when screensharing track ended --- src/webrtc/groupCall.ts | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index bb79d0441a2..e9fdc7e4088 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -419,6 +419,15 @@ export class GroupCall extends EventEmitter { const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + for (const track of stream.getTracks()) { + const onTrackEnded = () => { + this.setScreensharingEnabled(false); + track.removeEventListener("ended", onTrackEnded); + }; + + track.addEventListener("ended", onTrackEnded); + } + logger.log("Screensharing permissions granted. Setting screensharing enabled on all calls"); this.localDesktopCapturerSourceId = desktopCapturerSourceId; From 0683133d5bc81e515de945060ea3cf9cccadb7f9 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 19 Nov 2021 16:02:26 -0800 Subject: [PATCH 136/291] Dont start retry loop until weve sent the member state event --- src/webrtc/groupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e9fdc7e4088..02e6237abe9 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -271,13 +271,13 @@ export class GroupCall extends EventEmitter { for (const stateEvent of memberStateEvents) { this.onMemberStateChanged(stateEvent); } + + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); }); this.client.on("Call.incoming", this.onIncomingCall); this.onActiveSpeakerLoop(); - - this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); } private dispose() { From 96ef535ebba0d30e9db81f37acc896bacb9b563a Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 19 Nov 2021 16:06:29 -0800 Subject: [PATCH 137/291] Make unknown device error more useful --- src/webrtc/callEventHandler.ts | 2 +- src/webrtc/groupCall.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index fef13faa924..eb5cde0b779 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -191,7 +191,7 @@ export class CallEventHandler { GroupCallEvent.Error, new GroupCallError( GroupCallErrorCode.UnknownDevice, - `No opponent device found for ${senderId}, ignoring.`, + `Incoming Call: No opponent device found for ${senderId}, ignoring.`, ), ); return; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 02e6237abe9..eaeb69bf086 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -651,7 +651,7 @@ export class GroupCall extends EventEmitter { GroupCallEvent.Error, new GroupCallError( GroupCallErrorCode.UnknownDevice, - `No opponent device found for ${member.userId}, ignoring.`, + `Outgoing Call: No opponent device found for ${member.userId}, ignoring.`, ), ); return; From d0e37ee323c42296a6287878b076b989d22d4d28 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 19 Nov 2021 16:49:20 -0800 Subject: [PATCH 138/291] Hopefully resolve a race condition with missing device ids --- src/webrtc/groupCall.ts | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index eaeb69bf086..3626e9693b5 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -245,7 +245,7 @@ export class GroupCall extends EventEmitter { this.addParticipant(this.room.getMember(this.client.getUserId())); - const sendMemberStateEventPromise = this.sendMemberStateEvent(); + await this.sendMemberStateEvent(); this.activeSpeaker = null; @@ -253,6 +253,8 @@ export class GroupCall extends EventEmitter { logger.log(`Entered group call ${this.groupCallId}`); + this.client.on("Call.incoming", this.onIncomingCall); + const calls = this.client.callEventHandler.calls.values(); for (const call of calls) { @@ -264,18 +266,11 @@ export class GroupCall extends EventEmitter { const roomState = this.room.currentState; const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); - // This avoids a race condition where the other side would first receive - // the to-device messages and only then the member state event which - // would result in the call being ignored - sendMemberStateEventPromise.then(() => { - for (const stateEvent of memberStateEvents) { - this.onMemberStateChanged(stateEvent); - } - - this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); - }); + for (const stateEvent of memberStateEvents) { + this.onMemberStateChanged(stateEvent); + } - this.client.on("Call.incoming", this.onIncomingCall); + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); this.onActiveSpeakerLoop(); } From ffbd10a7b84b0dcaa277b6a15451858a21e37d30 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 22 Nov 2021 13:26:29 -0800 Subject: [PATCH 139/291] Make updateLocalUsermediaStreams stop tracks --- src/webrtc/call.ts | 4 ---- src/webrtc/mediaHandler.ts | 24 +++++++++++++++++++----- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 33afdfbcb34..12222d4ba68 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1095,8 +1095,6 @@ export class MatrixCall extends EventEmitter { * Request a new local usermedia stream with the current device id. */ public async updateLocalUsermediaStream(stream: MediaStream) { - const oldStream = this.localUsermediaStream; - const callFeed = this.localUsermediaFeed; callFeed.setNewStream(stream); const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; @@ -1140,8 +1138,6 @@ export class MatrixCall extends EventEmitter { } this.usermediaSenders = newSenders; - - this.client.getMediaHandler().stopUserMediaStream(oldStream); } /** diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 278a8b9c428..248e049bc5e 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -52,17 +52,31 @@ export class MediaHandler { } public async updateLocalUsermediaStreams(): Promise { + const callMediaStreamParams: Map = new Map(); + for (const call of this.client.callEventHandler.calls.values()) { + callMediaStreamParams.set(call.callId, { + audio: call.hasLocalUserMediaAudioTrack, + video: call.hasLocalUserMediaVideoTrack, + }); + } + + for (const stream of this.userMediaStreams) { + for (const track of stream.getTracks()) { + track.stop(); + } + } + + this.userMediaStreams = []; this.localUserMediaStream = undefined; for (const call of this.client.callEventHandler.calls.values()) { - if (call.state === CallState.Ended) { + if (call.state === CallState.Ended || !callMediaStreamParams.has(call.callId)) { continue; } - const stream = await this.getUserMediaStream( - call.hasLocalUserMediaAudioTrack, - call.hasLocalUserMediaVideoTrack, - ); + const { audio, video } = callMediaStreamParams.get(call.callId); + + const stream = await this.getUserMediaStream(audio, video); await call.updateLocalUsermediaStream(stream); } From d1a62eddfc5bd0f77ca990a18b5853afbf6abd0a Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 24 Nov 2021 12:43:50 -0800 Subject: [PATCH 140/291] Set initial audio/video input ids --- src/webrtc/mediaHandler.ts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 248e049bc5e..c6e16feb995 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -118,6 +118,17 @@ export class MediaHandler { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); logger.log("Getting user media with constraints", constraints); stream = await navigator.mediaDevices.getUserMedia(constraints); + + for (const track of stream.getTracks()) { + const settings = track.getSettings(); + + if (track.kind === "audio") { + this.audioInput = settings.deviceId; + } else if (track.kind === "video") { + this.videoInput = settings.deviceId; + } + } + this.localUserMediaStream = stream; } else { stream = this.localUserMediaStream.clone(); From fcc4b71f06344369a795494f540c593424339d78 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 29 Nov 2021 14:34:43 -0800 Subject: [PATCH 141/291] Add LocalStreamsChanged event to MediaHandler --- src/webrtc/mediaHandler.ts | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index c6e16feb995..ed522020909 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -17,19 +17,26 @@ See the License for the specific language governing permissions and limitations under the License. */ +import EventEmitter from "events"; import { GroupCallType } from "../webrtc/groupCall"; import { MatrixClient } from "../client"; import { logger } from "../logger"; import { CallState } from "./call"; -export class MediaHandler { +export enum MediaHandlerEvent { + LocalStreamsChanged = "local_streams_changed" +} + +export class MediaHandler extends EventEmitter { private audioInput: string; private videoInput: string; private localUserMediaStream?: MediaStream; public userMediaStreams: MediaStream[] = []; public screensharingStreams: MediaStream[] = []; - constructor(private client: MatrixClient) {} + constructor(private client: MatrixClient) { + super(); + } /** * Set an audio input device to use for MatrixCalls @@ -89,6 +96,8 @@ export class MediaHandler { await groupCall.updateLocalUsermediaStream(stream); } + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } public async hasAudioDevice(): Promise { @@ -148,6 +157,8 @@ export class MediaHandler { this.userMediaStreams.push(stream); + this.emit(MediaHandlerEvent.LocalStreamsChanged); + return stream; } @@ -166,6 +177,8 @@ export class MediaHandler { logger.debug("Splicing usermedia stream out stream array", mediaStream.id); this.userMediaStreams.splice(index, 1); } + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } /** @@ -195,6 +208,8 @@ export class MediaHandler { this.screensharingStreams.push(stream); + this.emit(MediaHandlerEvent.LocalStreamsChanged); + return stream; } @@ -213,6 +228,8 @@ export class MediaHandler { logger.debug("Splicing screensharing stream out stream array", mediaStream.id); this.screensharingStreams.splice(index, 1); } + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } /** @@ -234,6 +251,8 @@ export class MediaHandler { this.userMediaStreams = []; this.screensharingStreams = []; this.localUserMediaStream = undefined; + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints { From e336aceaba71a90277ad53db0beb65b364f2f36e Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 30 Nov 2021 13:01:35 -0800 Subject: [PATCH 142/291] Expose webrtc related types/props --- src/matrix.ts | 2 ++ src/webrtc/call.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/matrix.ts b/src/matrix.ts index ccdfea69ed0..1239ccdfdf2 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -48,12 +48,14 @@ export * from "./crypto/store/indexeddb-crypto-store"; export * from "./content-repo"; export * as ContentHelpers from "./content-helpers"; export { createNewMatrixCall } from "./webrtc/call"; +export type { MatrixCall } from "./webrtc/call"; export { GroupCallEvent, GroupCallIntent, GroupCallState, GroupCallType, } from "./webrtc/groupCall"; +export type { GroupCall } from "./webrtc/groupCall"; // expose the underlying request object so different environments can use // different request libs (e.g. request or browser-request) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 12222d4ba68..f42ab48b326 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -265,6 +265,7 @@ export class MatrixCall extends EventEmitter { public hangupReason: string; public direction: CallDirection; public ourPartyId: string; + public peerConn?: RTCPeerConnection; private client: MatrixClient; private forceTURN: boolean; @@ -275,7 +276,6 @@ export class MatrixCall extends EventEmitter { private candidateSendQueue: Array = []; private candidateSendTries = 0; private sentEndOfCandidates = false; - private peerConn: RTCPeerConnection; private feeds: Array = []; private usermediaSenders: Array = []; private screensharingSenders: Array = []; From 06d9d6207c637e0e437e03db3c3816c78dc6238a Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 30 Nov 2021 13:39:43 -0800 Subject: [PATCH 143/291] Send device id along with to device signaling messages --- src/webrtc/call.ts | 5 ++++- src/webrtc/callEventHandler.ts | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f42ab48b326..d698a4e6469 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1997,7 +1997,10 @@ export class MatrixCall extends EventEmitter { if (this.opponentDeviceId) { return this.client.sendToDevice(eventType, { [this.invitee || this.getOpponentMember().userId]: { - [this.opponentDeviceId]: realContent, + [this.opponentDeviceId]: { + ...realContent, + device_id: this.client.deviceId, + }, }, }); } else { diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index eb5cde0b779..4e34c3df3e8 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -183,7 +183,7 @@ export class CallEventHandler { return; } - opponentDeviceId = groupCall.getDeviceForMember(senderId)?.device_id; + opponentDeviceId = content.device_id; if (!opponentDeviceId) { logger.warn(`Cannot find a device id for ${senderId}. Ignoring event.`); From 549f9b7e295d3fab1eba974c48ccdde6b3bf472c Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 6 Jan 2022 14:44:16 -0800 Subject: [PATCH 144/291] Disable retries --- src/webrtc/call.ts | 8 ++++---- src/webrtc/groupCall.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d698a4e6469..fd569886c5a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1840,11 +1840,11 @@ export class MatrixCall extends EventEmitter { }, 1000); } } else if (this.peerConn.iceConnectionState == 'failed') { - this.peerConn.restartIce(); + //this.peerConn.restartIce(); } else if (this.peerConn.iceConnectionState == 'disconnected') { - this.iceDisconnectedTimeout = setTimeout(() => { - this.hangup(CallErrorCode.IceFailed, false); - }, 30 * 1000); + // this.iceDisconnectedTimeout = setTimeout(() => { + // this.hangup(CallErrorCode.IceFailed, false); + // }, 30 * 1000); } }; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3626e9693b5..19c7ae77b8d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -270,7 +270,7 @@ export class GroupCall extends EventEmitter { this.onMemberStateChanged(stateEvent); } - this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); + // this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); this.onActiveSpeakerLoop(); } From bef557976bd3071392d1b58e1b137ad54bd02b15 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 6 Jan 2022 15:24:59 -0800 Subject: [PATCH 145/291] Emit sent voip events --- src/webrtc/call.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index fd569886c5a..0dac41e04a7 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -134,6 +134,8 @@ export enum CallEvent { LengthChanged = 'length_changed', DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", } export enum CallErrorCode { @@ -1995,6 +1997,13 @@ export class MatrixCall extends EventEmitter { }); if (this.opponentDeviceId) { + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember().userId, + content: { ...realContent, device_id: this.client.deviceId }, + }); + return this.client.sendToDevice(eventType, { [this.invitee || this.getOpponentMember().userId]: { [this.opponentDeviceId]: { @@ -2004,6 +2013,13 @@ export class MatrixCall extends EventEmitter { }, }); } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + }); + return this.client.sendEvent(this.roomId, eventType, realContent); } } From 7993dd763011c6e3c2e5bbd6290fc5ee5966b8dd Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 6 Jan 2022 15:46:55 -0800 Subject: [PATCH 146/291] Log opponentDeviceId --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0dac41e04a7..97c4a3ddb79 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2001,6 +2001,7 @@ export class MatrixCall extends EventEmitter { type: "toDevice", eventType, userId: this.invitee || this.getOpponentMember().userId, + opponentDeviceId: this.opponentDeviceId, content: { ...realContent, device_id: this.client.deviceId }, }); From f3f9e41787358b8c3e10be4f206cc20654287701 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 6 Jan 2022 15:24:59 -0800 Subject: [PATCH 147/291] Emit sent voip events --- src/webrtc/call.ts | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d698a4e6469..df92850daa1 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -134,6 +134,8 @@ export enum CallEvent { LengthChanged = 'length_changed', DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", } export enum CallErrorCode { @@ -1995,6 +1997,13 @@ export class MatrixCall extends EventEmitter { }); if (this.opponentDeviceId) { + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember().userId, + content: { ...realContent, device_id: this.client.deviceId }, + }); + return this.client.sendToDevice(eventType, { [this.invitee || this.getOpponentMember().userId]: { [this.opponentDeviceId]: { @@ -2004,6 +2013,13 @@ export class MatrixCall extends EventEmitter { }, }); } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + }); + return this.client.sendEvent(this.roomId, eventType, realContent); } } From 18bb5c3079e42ad3594117fba8d12f146133df74 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 6 Jan 2022 15:46:55 -0800 Subject: [PATCH 148/291] Log opponentDeviceId --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index df92850daa1..1ce5d4e0b2f 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2001,6 +2001,7 @@ export class MatrixCall extends EventEmitter { type: "toDevice", eventType, userId: this.invitee || this.getOpponentMember().userId, + opponentDeviceId: this.opponentDeviceId, content: { ...realContent, device_id: this.client.deviceId }, }); From 87bf1159673d36b810eb0090297b01239f53b4ca Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 10 Jan 2022 15:57:40 -0800 Subject: [PATCH 149/291] Use session ids to resolve refresh during invite/answer --- src/client.ts | 10 ++++++++++ src/webrtc/call.ts | 9 ++++++++- src/webrtc/callEventTypes.ts | 2 ++ src/webrtc/groupCall.ts | 20 +++++++++++++------- 4 files changed, 33 insertions(+), 8 deletions(-) diff --git a/src/client.ts b/src/client.ts index fd162128d53..917b4215ccc 100644 --- a/src/client.ts +++ b/src/client.ts @@ -758,6 +758,7 @@ export class MatrixClient extends EventEmitter { protected exportedOlmDeviceToImport: IOlmDevice; protected txnCtr = 0; protected mediaHandler = new MediaHandler(this); + protected sessionId: string; constructor(opts: IMatrixClientCreateOpts) { super(); @@ -771,6 +772,7 @@ export class MatrixClient extends EventEmitter { this.usingExternalCrypto = opts.usingExternalCrypto; this.store = opts.store || new StubStore(); this.deviceId = opts.deviceId || null; + this.sessionId = randomString(10); const userId = opts.userId || null; this.credentials = { userId }; @@ -1259,6 +1261,14 @@ export class MatrixClient extends EventEmitter { return this.deviceId; } + /** + * Get the session ID of this client + * @return {string} session ID + */ + public getSessionId(): string { + return this.sessionId; + } + /** * Check if the runtime environment supports VoIP calling. * @return {boolean} True if VoIP is supported. diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1ce5d4e0b2f..e0b33da8077 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -319,6 +319,7 @@ export class MatrixCall extends EventEmitter { private callLength = 0; private opponentDeviceId: string; + private opponentSessionId: string; public groupCallId: string; constructor(opts: CallOpts) { @@ -374,6 +375,10 @@ export class MatrixCall extends EventEmitter { return this.opponentMember; } + public getOpponentSessionId(): string { + return this.opponentSessionId; + } + public opponentCanBeTransferred(): boolean { return Boolean(this.opponentCaps && this.opponentCaps["m.call.transferee"]); } @@ -2002,7 +2007,7 @@ export class MatrixCall extends EventEmitter { eventType, userId: this.invitee || this.getOpponentMember().userId, opponentDeviceId: this.opponentDeviceId, - content: { ...realContent, device_id: this.client.deviceId }, + content: { ...realContent, device_id: this.client.deviceId, session_id: this.client.getSessionId() }, }); return this.client.sendToDevice(eventType, { @@ -2010,6 +2015,7 @@ export class MatrixCall extends EventEmitter { [this.opponentDeviceId]: { ...realContent, device_id: this.client.deviceId, + session_id: this.client.getSessionId(), }, }, }); @@ -2339,6 +2345,7 @@ export class MatrixCall extends EventEmitter { } this.opponentCaps = msg.capabilities || {} as CallCapabilities; this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); + this.opponentSessionId = msg.session_id; } private async addBufferedIceCandidates(): Promise { diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index c5b0063f9d7..28eeb6d940f 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -36,6 +36,7 @@ export interface MCallBase { call_id: string; version: string | number; party_id?: string; + session_id?: string; } export interface MCallAnswer extends MCallBase { @@ -54,6 +55,7 @@ export interface MCallInviteNegotiate extends MCallBase { lifetime: number; capabilities?: CallCapabilities; invitee?: string; + session_id?: string; [SDPStreamMetadataKey]: SDPStreamMetadata; } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3626e9693b5..8ca0ce7673a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -73,6 +73,7 @@ export interface IGroupCallRoomMemberFeed { export interface IGroupCallRoomMemberDevice { "device_id": string; + "session_id": string; "feeds": IGroupCallRoomMemberFeed[]; } @@ -532,6 +533,7 @@ export class GroupCall extends EventEmitter { "m.devices": [ { "device_id": deviceId, + "session_id": this.client.getSessionId(), "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose, })), @@ -632,12 +634,6 @@ export class GroupCall extends EventEmitter { return; } - const existingCall = this.getCallByUserId(member.userId); - - if (existingCall) { - return; - } - const opponentDevice = this.getDeviceForMember(member.userId); if (!opponentDevice) { @@ -652,6 +648,12 @@ export class GroupCall extends EventEmitter { return; } + const existingCall = this.getCallByUserId(member.userId); + + if (existingCall && existingCall.getOpponentSessionId() === opponentDevice.session_id) { + return; + } + const newCall = createNewMatrixCall( this.client, this.room.roomId, @@ -668,7 +670,11 @@ export class GroupCall extends EventEmitter { newCall.createDataChannel("datachannel", this.dataChannelOptions); } - this.addCall(newCall); + if (existingCall) { + this.replaceCall(existingCall, newCall); + } else { + this.addCall(newCall); + } }; public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { From c91617a79925e67e6bcb3fa83119103c055df626 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 10 Jan 2022 16:22:52 -0800 Subject: [PATCH 150/291] Force hangup replaced calls --- src/webrtc/groupCall.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 8ca0ce7673a..37efb256821 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -671,7 +671,7 @@ export class GroupCall extends EventEmitter { } if (existingCall) { - this.replaceCall(existingCall, newCall); + this.replaceCall(existingCall, newCall, true); } else { this.addCall(newCall); } @@ -734,7 +734,7 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.CallsChanged, this.calls); } - private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall) { + private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, forceHangup = false) { const existingCallIndex = this.calls.indexOf(existingCall); if (existingCallIndex === -1) { @@ -743,7 +743,7 @@ export class GroupCall extends EventEmitter { this.calls.splice(existingCallIndex, 1, replacementCall); - this.disposeCall(existingCall, CallErrorCode.Replaced); + this.disposeCall(existingCall, CallErrorCode.Replaced, forceHangup); this.initCall(replacementCall); this.emit(GroupCallEvent.CallsChanged, this.calls); @@ -793,7 +793,7 @@ export class GroupCall extends EventEmitter { onCallFeedsChanged(); } - private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { + private disposeCall(call: MatrixCall, hangupReason: CallErrorCode, forceHangup = false) { const opponentMemberId = getCallUserId(call); if (!opponentMemberId) { @@ -814,7 +814,7 @@ export class GroupCall extends EventEmitter { this.callHandlers.delete(opponentMemberId); - if (call.hangupReason === CallErrorCode.Replaced) { + if (call.hangupReason === CallErrorCode.Replaced && !forceHangup) { return; } From 3fac6d7180b452014d177c5512add5efe5bee5bd Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 10 Jan 2022 16:46:55 -0800 Subject: [PATCH 151/291] Replace outbound calls only --- src/webrtc/groupCall.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index cdd1d6bc92a..73287d58d77 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,7 +1,7 @@ import EventEmitter from "events"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient } from "../client"; -import { CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; +import { CallDirection, CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; @@ -650,7 +650,11 @@ export class GroupCall extends EventEmitter { const existingCall = this.getCallByUserId(member.userId); - if (existingCall && existingCall.getOpponentSessionId() === opponentDevice.session_id) { + if ( + existingCall && + existingCall.getOpponentSessionId() === opponentDevice.session_id && + existingCall.direction === CallDirection.Inbound + ) { return; } @@ -671,6 +675,7 @@ export class GroupCall extends EventEmitter { } if (existingCall) { + console.log("!!!!!!!!!!!!!!!!!!!! IT WORKED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); this.replaceCall(existingCall, newCall, true); } else { this.addCall(newCall); From 7529af43e46b16bdf583f9244f3b164d49e9b16b Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 11 Jan 2022 16:54:12 -0800 Subject: [PATCH 152/291] Add NewSession CallErrorCode --- src/webrtc/call.ts | 5 +++++ src/webrtc/groupCall.ts | 13 ++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bbce1343be3..ba5909ba253 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -217,6 +217,11 @@ export enum CallErrorCode { * We transferred the call off to somewhere else */ Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', } /** diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 73287d58d77..efb9f994dc0 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -512,7 +512,7 @@ export class GroupCall extends EventEmitter { // Check if the user calling has an existing call and use this call instead. if (existingCall) { - this.replaceCall(existingCall, newCall); + this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); } else { this.addCall(newCall); } @@ -675,8 +675,7 @@ export class GroupCall extends EventEmitter { } if (existingCall) { - console.log("!!!!!!!!!!!!!!!!!!!! IT WORKED !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!"); - this.replaceCall(existingCall, newCall, true); + this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); } else { this.addCall(newCall); } @@ -739,7 +738,7 @@ export class GroupCall extends EventEmitter { this.emit(GroupCallEvent.CallsChanged, this.calls); } - private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, forceHangup = false) { + private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, hangupReason = CallErrorCode.Replaced) { const existingCallIndex = this.calls.indexOf(existingCall); if (existingCallIndex === -1) { @@ -748,7 +747,7 @@ export class GroupCall extends EventEmitter { this.calls.splice(existingCallIndex, 1, replacementCall); - this.disposeCall(existingCall, CallErrorCode.Replaced, forceHangup); + this.disposeCall(existingCall, hangupReason); this.initCall(replacementCall); this.emit(GroupCallEvent.CallsChanged, this.calls); @@ -798,7 +797,7 @@ export class GroupCall extends EventEmitter { onCallFeedsChanged(); } - private disposeCall(call: MatrixCall, hangupReason: CallErrorCode, forceHangup = false) { + private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { const opponentMemberId = getCallUserId(call); if (!opponentMemberId) { @@ -819,7 +818,7 @@ export class GroupCall extends EventEmitter { this.callHandlers.delete(opponentMemberId); - if (call.hangupReason === CallErrorCode.Replaced && !forceHangup) { + if (call.hangupReason === CallErrorCode.Replaced) { return; } From ee995cb39bfadf576c7d3f040e6a5d60e1d58def Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 11 Jan 2022 16:54:40 -0800 Subject: [PATCH 153/291] Ensure call events are processed once and in order --- src/webrtc/callEventHandler.ts | 120 +++++++++++---------------------- 1 file changed, 39 insertions(+), 81 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 4e34c3df3e8..e2dae24c7ce 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -20,7 +20,6 @@ import { createNewMatrixCall, MatrixCall, CallErrorCode, CallState, CallDirectio import { EventType } from '../@types/event'; import { MatrixClient } from '../client'; import { MCallAnswer, MCallHangupReject } from "./callEventTypes"; -import { SyncState } from "../sync.api"; import { GroupCallError, GroupCallErrorCode, GroupCallEvent } from './groupCall'; // Don't ring unless we'd be ringing for at least 3 seconds: the user needs some @@ -33,7 +32,7 @@ export class CallEventHandler { callEventBuffer: MatrixEvent[]; candidateEventsByCall: Map>; - private toDeviceCallEventBuffer: MatrixEvent[] = []; + private eventBufferPromiseChain?: Promise; constructor(client: MatrixClient) { this.client = client; @@ -63,101 +62,60 @@ export class CallEventHandler { } private onSync = (): void => { - this.evaluateEventBuffer(); - this.evaluateToDeviceEventBuffer(); - }; + // Process the current event buffer and start queuing into a new one. + const currentEventBuffer = this.callEventBuffer; + this.callEventBuffer = []; - private async evaluateEventBuffer() { - if (this.client.getSyncState() === SyncState.Syncing) { - await Promise.all(this.callEventBuffer.map(event => { - this.client.decryptEventIfNeeded(event); - })); - - const ignoreCallIds = new Set(); - // inspect the buffer and mark all calls which have been answered - // or hung up before passing them to the call event handler. - for (const ev of this.callEventBuffer) { - if (ev.getType() === EventType.CallAnswer || - ev.getType() === EventType.CallHangup) { - ignoreCallIds.add(ev.getContent().call_id); - } - } - // now loop through the buffer chronologically and inject them - for (const e of this.callEventBuffer) { - if ( - e.getType() === EventType.CallInvite && - ignoreCallIds.has(e.getContent().call_id) - ) { - // This call has previously been answered or hung up: ignore it - continue; - } - try { - await this.handleCallEvent(e); - } catch (e) { - logger.error("Caught exception handling call event", e); - } - } - this.callEventBuffer = []; + // Ensure correct ordering by only processing this queue after the previous one has finished processing + if (this.eventBufferPromiseChain) { + this.eventBufferPromiseChain = + this.eventBufferPromiseChain.then(() => this.evaluateEventBuffer(currentEventBuffer)); + } else { + this.eventBufferPromiseChain = this.evaluateEventBuffer(currentEventBuffer); } - } + }; private onRoomTimeline = (event: MatrixEvent) => { - this.client.decryptEventIfNeeded(event); - // any call events or ones that might be once they're decrypted - if (this.eventIsACall(event) || event.isBeingDecrypted()) { - // queue up for processing once all events from this sync have been - // processed (see above). - this.callEventBuffer.push(event); - } - - if (event.isBeingDecrypted() || event.isDecryptionFailure()) { - // add an event listener for once the event is decrypted. - event.once("Event.decrypted", async () => { - if (!this.eventIsACall(event)) return; - - if (this.callEventBuffer.includes(event)) { - // we were waiting for that event to decrypt, so recheck the buffer - this.evaluateEventBuffer(); - } else { - // This one wasn't buffered so just run the event handler for it - // straight away - try { - await this.handleCallEvent(event); - } catch (e) { - logger.error("Caught exception handling call event", e); - } - } - }); - } + this.callEventBuffer.push(event); }; private onToDeviceEvent = (event: MatrixEvent): void => { - if (!this.eventIsACall(event)) return; - - this.toDeviceCallEventBuffer.push(event); + this.callEventBuffer.push(event); }; - private async evaluateToDeviceEventBuffer(): Promise { - if (this.client.getSyncState() !== SyncState.Syncing) return; + private async evaluateEventBuffer(eventBuffer: MatrixEvent[]) { + await Promise.all(eventBuffer.map((event) => this.client.decryptEventIfNeeded(event))); + + const callEvents = eventBuffer.filter((event) => event.getType().startsWith("m.call.")); + + const ignoreCallIds = new Set(); + + // inspect the buffer and mark all calls which have been answered + // or hung up before passing them to the call event handler. + for (const event of callEvents) { + const eventType = event.getType(); + + if (eventType=== EventType.CallAnswer || eventType === EventType.CallHangup) { + ignoreCallIds.add(event.getContent().call_id); + } + } + + // Process call events in the order that they were received + for (const event of callEvents) { + const eventType = event.getType(); + const callId = event.getContent().call_id; + + if (eventType === EventType.CallInvite && ignoreCallIds.has(callId)) { + // This call has previously been answered or hung up: ignore it + continue; + } - for (const event of this.toDeviceCallEventBuffer) { try { await this.handleCallEvent(event); } catch (e) { logger.error("Caught exception handling call event", e); } } - - this.toDeviceCallEventBuffer = []; - } - - private eventIsACall(event: MatrixEvent): boolean { - const type = event.getType(); - /** - * Unstable prefixes: - * - org.matrix.call. : MSC3086 https://github.com/matrix-org/matrix-doc/pull/3086 - */ - return type.startsWith("m.call.") || type.startsWith("org.matrix.call."); } private async handleCallEvent(event: MatrixEvent) { From eeacf8c22cd4acf8d7b29313e05d22a9177c7855 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 11 Jan 2022 17:47:01 -0800 Subject: [PATCH 154/291] Dont filter unstable call events --- src/webrtc/callEventHandler.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index e2dae24c7ce..e44b91f1316 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -86,7 +86,10 @@ export class CallEventHandler { private async evaluateEventBuffer(eventBuffer: MatrixEvent[]) { await Promise.all(eventBuffer.map((event) => this.client.decryptEventIfNeeded(event))); - const callEvents = eventBuffer.filter((event) => event.getType().startsWith("m.call.")); + const callEvents = eventBuffer.filter((event) => { + const eventType = event.getType(); + return eventType.startsWith("m.call.") || eventType.startsWith("org.matrix.call."); + }); const ignoreCallIds = new Set(); From d8285aad00cc4632c6303811bb05f2c5c7645efb Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 12 Jan 2022 13:17:03 -0800 Subject: [PATCH 155/291] Remove call from callEventHandler after hangup --- src/webrtc/call.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ba5909ba253..4a90dde0cc0 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2161,6 +2161,8 @@ export class MatrixCall extends EventEmitter { if (shouldEmit) { this.emit(CallEvent.Hangup, this); } + + this.client.callEventHandler.calls.delete(this.callId); } private stopAllMedia(): void { From 28f3169a28ac1b6ef27c0678e3b4e9a917ae732c Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 12 Jan 2022 13:17:19 -0800 Subject: [PATCH 156/291] Use replace error code when replacing incoming calls --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index efb9f994dc0..cb689f34edb 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -512,7 +512,7 @@ export class GroupCall extends EventEmitter { // Check if the user calling has an existing call and use this call instead. if (existingCall) { - this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); + this.replaceCall(existingCall, newCall); } else { this.addCall(newCall); } From 42fef0e7aa11e7a3540f2b4a86201ab0051d1a9b Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 13 Jan 2022 14:10:39 -0800 Subject: [PATCH 157/291] Add user id to all send voip events --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4a90dde0cc0..48fd03a16c9 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2030,6 +2030,7 @@ export class MatrixCall extends EventEmitter { eventType, roomId: this.roomId, content: realContent, + userId: this.invitee || this.getOpponentMember().userId, }); return this.client.sendEvent(this.roomId, eventType, realContent); From 015eb5d5c4c30d11d065a911b1f1737256f5f88e Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 14 Jan 2022 13:40:42 -0800 Subject: [PATCH 158/291] Add sender/dest session ids --- src/webrtc/call.ts | 16 +++++++++++++--- src/webrtc/callEventHandler.ts | 18 ++++++++++++++++-- src/webrtc/callEventTypes.ts | 6 ++++-- src/webrtc/groupCall.ts | 12 ++++++++---- 4 files changed, 41 insertions(+), 11 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 48fd03a16c9..17614dc63aa 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -72,6 +72,7 @@ interface CallOpts { forceTURN?: boolean; turnServers?: Array; opponentDeviceId?: string; + opponentSessionId?: string; groupCallId?: string; } @@ -335,6 +336,7 @@ export class MatrixCall extends EventEmitter { this.forceTURN = opts.forceTURN; this.ourPartyId = this.client.deviceId; this.opponentDeviceId = opts.opponentDeviceId; + this.opponentSessionId = opts.opponentSessionId; this.groupCallId = opts.groupCallId; // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; @@ -2007,12 +2009,19 @@ export class MatrixCall extends EventEmitter { }); if (this.opponentDeviceId) { + logger.log(`sendVoipEvent localSessionId: ${ + this.client.getSessionId()} destSessionId: ${this.opponentSessionId}`); this.emit(CallEvent.SendVoipEvent, { type: "toDevice", eventType, userId: this.invitee || this.getOpponentMember().userId, opponentDeviceId: this.opponentDeviceId, - content: { ...realContent, device_id: this.client.deviceId, session_id: this.client.getSessionId() }, + content: { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + }, }); return this.client.sendToDevice(eventType, { @@ -2020,7 +2029,8 @@ export class MatrixCall extends EventEmitter { [this.opponentDeviceId]: { ...realContent, device_id: this.client.deviceId, - session_id: this.client.getSessionId(), + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, }, }, }); @@ -2353,7 +2363,6 @@ export class MatrixCall extends EventEmitter { } this.opponentCaps = msg.capabilities || {} as CallCapabilities; this.opponentMember = this.client.getRoom(this.roomId).getMember(ev.getSender()); - this.opponentSessionId = msg.session_id; } private async addBufferedIceCandidates(): Promise { @@ -2450,6 +2459,7 @@ export function createNewMatrixCall(client: any, roomId: string, options?: CallO // call level options forceTURN: client.forceTURN || optionsForceTURN, opponentDeviceId: options?.opponentDeviceId, + opponentSessionId: options?.opponentSessionId, groupCallId: options?.groupCallId, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index e44b91f1316..ecbbd51f94f 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -157,6 +157,11 @@ export class CallEventHandler { ); return; } + + if (content.dest_session_id !== this.client.getSessionId()) { + logger.warn("Call event does not match current session id, ignoring."); + return; + } } if (!callRoomId) return; @@ -185,7 +190,11 @@ export class CallEventHandler { call = createNewMatrixCall( this.client, callRoomId, - { forceTURN: this.client.forceTURN, opponentDeviceId, groupCallId }, + { + forceTURN: this.client.forceTURN, opponentDeviceId, + groupCallId, + opponentSessionId: content.sender_session_id, + }, ); if (!call) { logger.log( @@ -266,7 +275,12 @@ export class CallEventHandler { // we're probably getting events backwards so // the hangup will come before the invite call = createNewMatrixCall( - this.client, callRoomId, { opponentDeviceId }, + this.client, + callRoomId, + { + opponentDeviceId, + opponentSessionId: content.sender_session_id, + }, ); if (call) { call.callId = content.call_id; diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index 28eeb6d940f..4f43a70a1d5 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -36,7 +36,8 @@ export interface MCallBase { call_id: string; version: string | number; party_id?: string; - session_id?: string; + sender_session_id?: string; + dest_session_id?: string; } export interface MCallAnswer extends MCallBase { @@ -55,7 +56,8 @@ export interface MCallInviteNegotiate extends MCallBase { lifetime: number; capabilities?: CallCapabilities; invitee?: string; - session_id?: string; + sender_session_id?: string; + dest_session_id?: string; [SDPStreamMetadataKey]: SDPStreamMetadata; } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index cb689f34edb..d22cae6e713 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,7 +1,7 @@ import EventEmitter from "events"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient } from "../client"; -import { CallDirection, CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; +import { CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; @@ -652,8 +652,7 @@ export class GroupCall extends EventEmitter { if ( existingCall && - existingCall.getOpponentSessionId() === opponentDevice.session_id && - existingCall.direction === CallDirection.Inbound + existingCall.getOpponentSessionId() === opponentDevice.session_id ) { return; } @@ -661,7 +660,12 @@ export class GroupCall extends EventEmitter { const newCall = createNewMatrixCall( this.client, this.room.roomId, - { invitee: member.userId, opponentDeviceId: opponentDevice.device_id, groupCallId: this.groupCallId }, + { + invitee: member.userId, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + }, ); const requestScreenshareFeed = opponentDevice.feeds.some( From 592fb0cf102732f43127b2e8f08af757170d5d79 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 18 Jan 2022 10:37:49 -0800 Subject: [PATCH 159/291] Re-enable retries --- src/webrtc/call.ts | 8 ++++---- src/webrtc/groupCall.ts | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 17614dc63aa..d3cc9aad0a5 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1854,11 +1854,11 @@ export class MatrixCall extends EventEmitter { }, 1000); } } else if (this.peerConn.iceConnectionState == 'failed') { - //this.peerConn.restartIce(); + this.peerConn.restartIce(); } else if (this.peerConn.iceConnectionState == 'disconnected') { - // this.iceDisconnectedTimeout = setTimeout(() => { - // this.hangup(CallErrorCode.IceFailed, false); - // }, 30 * 1000); + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); } }; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index d22cae6e713..56bb8938553 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -271,7 +271,7 @@ export class GroupCall extends EventEmitter { this.onMemberStateChanged(stateEvent); } - // this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); this.onActiveSpeakerLoop(); } From bbf70207552825d87b75d8d0b76b127dce2c264a Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 18 Jan 2022 10:39:02 -0800 Subject: [PATCH 160/291] Remove log --- src/webrtc/call.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index d3cc9aad0a5..87a87803011 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2009,8 +2009,6 @@ export class MatrixCall extends EventEmitter { }); if (this.opponentDeviceId) { - logger.log(`sendVoipEvent localSessionId: ${ - this.client.getSessionId()} destSessionId: ${this.opponentSessionId}`); this.emit(CallEvent.SendVoipEvent, { type: "toDevice", eventType, From 1a78301adb720646eca34b669dcc14bf05f18ff6 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 4 Feb 2022 12:11:37 -0800 Subject: [PATCH 161/291] Fix restartIce on FF Android --- src/webrtc/call.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 87a87803011..942d76ee457 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1854,7 +1854,12 @@ export class MatrixCall extends EventEmitter { }, 1000); } } else if (this.peerConn.iceConnectionState == 'failed') { - this.peerConn.restartIce(); + // Firefox for Android does not yet have support for restartIce() + if (this.peerConn.restartIce) { + this.peerConn.restartIce(); + } else { + this.hangup(CallErrorCode.IceFailed, false); + } } else if (this.peerConn.iceConnectionState == 'disconnected') { this.iceDisconnectedTimeout = setTimeout(() => { this.hangup(CallErrorCode.IceFailed, false); From 4a4465b9fc3ff82dba3725003e905248a02a01c3 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 7 Feb 2022 14:42:07 -0800 Subject: [PATCH 162/291] Don't send candidates after the call has ended --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 942d76ee457..ea7bddcb71d 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2211,7 +2211,7 @@ export class MatrixCall extends EventEmitter { } private async sendCandidateQueue(): Promise { - if (this.candidateSendQueue.length === 0) { + if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { return; } From d42e2fe2c0380d694642c47ab58b3436991e6f02 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 10 Feb 2022 09:32:34 -0800 Subject: [PATCH 163/291] Ignore duplicate streams when adding local feeds --- src/webrtc/call.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ea7bddcb71d..e380b8ec59a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -630,6 +630,11 @@ export class MatrixCall extends EventEmitter { * @param {boolean} addToPeerConnection whether to add the tracks to the peer connection */ public pushLocalFeed(callFeed: CallFeed, addToPeerConnection = true): void { + if (this.feeds.some((feed) => callFeed.stream.id === feed.stream.id)) { + logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + return; + } + this.feeds.push(callFeed); if (addToPeerConnection) { From b4fe00a3a89900791dce21f23a56b67c0b5a5bef Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 10 Feb 2022 10:31:52 -0800 Subject: [PATCH 164/291] Add answer/negotiate response promise chain --- src/webrtc/call.ts | 56 +++++++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e380b8ec59a..8c39cd96ab8 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -312,6 +312,8 @@ export class MatrixCall extends EventEmitter { private makingOffer = false; private ignoreOffer: boolean; + private responsePromiseChain?: Promise; + // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers // the call) we buffer them up here so we can then add the ones from the party we pick @@ -895,7 +897,7 @@ export class MatrixCall extends EventEmitter { logger.debug(`Answering call ${this.callId}`); - this.gotCallFeedsForAnswer(callFeeds); + this.queueGotCallFeedsForAnswer(callFeeds); } /** @@ -909,7 +911,7 @@ export class MatrixCall extends EventEmitter { newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { logger.debug("Handing local stream to new call"); - newCall.gotCallFeedsForAnswer(this.getLocalFeeds()); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds()); } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -1296,7 +1298,7 @@ export class MatrixCall extends EventEmitter { private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { if (this.successor) { - this.successor.gotCallFeedsForAnswer(callFeeds); + this.successor.queueGotCallFeedsForAnswer(callFeeds); return; } if (this.callHasEnded()) { @@ -1367,6 +1369,16 @@ export class MatrixCall extends EventEmitter { this.sendCandidateQueue(); } + private queueGotCallFeedsForAnswer(callFeeds: CallFeed[]): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.gotCallFeedsForAnswer(callFeeds)); + } else { + this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); + } + } + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { if (this.callHasEnded()) return; @@ -1714,7 +1726,30 @@ export class MatrixCall extends EventEmitter { return this.state === CallState.Ended; } - private gotLocalOffer = async (): Promise => { + private queueGotLocalOffer(): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); + } else { + this.responsePromiseChain = this.wrappedGotLocalOffer(); + } + } + + private async wrappedGotLocalOffer(): Promise { + this.makingOffer = true; + try { + this.getRidOfRTXCodecs(); + await this.gotLocalOffer(); + } catch (e) { + this.getLocalOfferFailed(e); + return; + } finally { + this.makingOffer = false; + } + } + + private async gotLocalOffer(): Promise { logger.debug("Setting local description"); if (this.callHasEnded()) { @@ -1805,7 +1840,7 @@ export class MatrixCall extends EventEmitter { } }, CALL_TIMEOUT_MS); } - }; + } private getLocalOfferFailed = (err: Error): void => { logger.error("Failed to get local offer", err); @@ -1946,16 +1981,7 @@ export class MatrixCall extends EventEmitter { return; } - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - await this.gotLocalOffer(); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } + this.queueGotLocalOffer(); }; public onHangupReceived = (msg: MCallHangupReject): void => { From c801690e283a22f6f2b2e6f7ecc683bb0eb2fd68 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 11 Feb 2022 16:56:47 -0800 Subject: [PATCH 165/291] Don't reuse local call feeds that have been added to a RTCPeerConnection --- src/webrtc/call.ts | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8c39cd96ab8..eb6d1c7b3f3 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -910,8 +910,12 @@ export class MatrixCall extends EventEmitter { logger.debug("Telling new call to wait for local media"); newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { - logger.debug("Handing local stream to new call"); - newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds()); + if (newCall.direction === CallDirection.Outbound) { + newCall.queueGotCallFeedsForAnswer([]); + } else { + logger.debug("Handing local stream to new call"); + newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); + } } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -2196,8 +2200,7 @@ export class MatrixCall extends EventEmitter { this.callStatsAtEnd = await this.collectCallStats(); // Order is important here: first we stopAllMedia() and only then we can deleteAllFeeds() - // We don't stop media if the call was replaced as we want to re-use streams in the successor - if (hangupReason !== CallErrorCode.Replaced) this.stopAllMedia(); + this.stopAllMedia(); this.deleteAllFeeds(); if (this.peerConn && this.peerConn.signalingState !== 'closed') { From d8e597ccdfb18bf307c8a6c5cc7b064d23eb1acf Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 11 Feb 2022 17:01:22 -0800 Subject: [PATCH 166/291] Avoid glare --- src/webrtc/groupCall.ts | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 56bb8938553..18c3f9ba08c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -634,6 +634,13 @@ export class GroupCall extends EventEmitter { return; } + // Only initiate a call with a user who has a userId that is lexicographically + // less than your own. Otherwise, that user will call you. + if (member.userId < localUserId) { + logger.log(`Waiting for ${member.userId} to send call invite.`); + return; + } + const opponentDevice = this.getDeviceForMember(member.userId); if (!opponentDevice) { From d12bccd211a4540bba72e5b389f298537500b0b5 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 15 Feb 2022 10:51:22 -0800 Subject: [PATCH 167/291] Remove safari hack --- src/webrtc/call.ts | 63 ---------------------------------------------- 1 file changed, 63 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index eb6d1c7b3f3..8eacdb7ec6b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1419,69 +1419,6 @@ export class MatrixCall extends EventEmitter { this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } - - // HACK: Safari doesn't like it when we reuse MediaStreams. In most cases - // we can get around this by calling MediaStream.clone(), however inbound - // calls seem to still be broken unless we getUserMedia again and replace - // all MediaStreams using sender.replaceTrack - if (isSafari) { - await new Promise(resolve => { - setTimeout(resolve, 200); - }); - - if (this.state === CallState.Ended) { - return; - } - - const callFeed = this.localUsermediaFeed; - const stream = callFeed.stream; - - if (!stream.active) { - throw new Error(`Call ${this.callId} has an inactive stream ${ - stream.id} and its tracks cannot be replaced`); - } - - const newSenders = []; - - for (const track of this.localUsermediaStream.getTracks()) { - const oldSender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === track.kind; - }); - - if (track.readyState === "ended") { - throw new Error(`Call ${this.callId} tried to replace track ${track.id} in the ended state`); - } - - let newSender: RTCRtpSender; - - try { - logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { - logger.info( - `Adding track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - newSender = this.peerConn.addTrack(track, stream); - } - - newSenders.push(newSender); - } - - this.usermediaSenders = newSenders; - } } /** From a2a127d9a473e5c166e5093161145bf5e6cbee34 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 15 Feb 2022 10:53:28 -0800 Subject: [PATCH 168/291] Remove unused isSafari check --- src/webrtc/call.ts | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8eacdb7ec6b..bf77de6a9dc 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -236,8 +236,6 @@ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; /** The length of time a call can be ringing for. */ const CALL_TIMEOUT_MS = 60000; -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - export class CallError extends Error { code: string; From 3db056ad3ea195bb1be26fdd94f8093067d513be Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 16 Feb 2022 19:06:46 +0000 Subject: [PATCH 169/291] Enable max-bundle (#2182) No particular reason to worry about old user agents here --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bf77de6a9dc..0b1fcd42025 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2291,6 +2291,7 @@ export class MatrixCall extends EventEmitter { iceTransportPolicy: this.forceTURN ? 'relay' : undefined, iceServers: this.turnServers, iceCandidatePoolSize: this.client.iceCandidatePoolSize, + bundlePolicy: 'max-bundle', }); // 'connectionstatechange' would be better, but firefox doesn't implement that. From fa5eae70dd41a04cc81c6830bde2a60b7ee1a7a0 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 17 Feb 2022 14:07:21 -0800 Subject: [PATCH 170/291] Log complete sync errors --- src/sync.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sync.ts b/src/sync.ts index 2a0030bffc0..3d58188d9c9 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -821,7 +821,7 @@ export class SyncApi { } catch (e) { // log the exception with stack if we have it, else fall back // to the plain description - logger.error("Caught /sync error", e.stack || e); + logger.error(e); // Emit the exception for client handling this.client.emit("sync.unexpectedError", e); From 7f21f569d53d0ec480176ee5f5fcaefd3ea7f8de Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 17 Feb 2022 14:08:17 -0800 Subject: [PATCH 171/291] Process toDevice events in order --- src/webrtc/call.ts | 5 ++++ src/webrtc/callEventHandler.ts | 44 +++++++++++++++++++++++++++++++++- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0b1fcd42025..bb00a8c9ccc 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -272,6 +272,7 @@ export class MatrixCall extends EventEmitter { public direction: CallDirection; public ourPartyId: string; public peerConn?: RTCPeerConnection; + public toDeviceSeq = 0; private client: MatrixClient; private forceTURN: boolean; @@ -1984,6 +1985,8 @@ export class MatrixCall extends EventEmitter { }); if (this.opponentDeviceId) { + const toDeviceSeq = this.toDeviceSeq++; + this.emit(CallEvent.SendVoipEvent, { type: "toDevice", eventType, @@ -1994,6 +1997,7 @@ export class MatrixCall extends EventEmitter { device_id: this.client.deviceId, sender_session_id: this.client.getSessionId(), dest_session_id: this.opponentSessionId, + seq: toDeviceSeq, }, }); @@ -2004,6 +2008,7 @@ export class MatrixCall extends EventEmitter { device_id: this.client.deviceId, sender_session_id: this.client.getSessionId(), dest_session_id: this.opponentSessionId, + seq: toDeviceSeq, }, }, }); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index ecbbd51f94f..ea454a4bd97 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -31,6 +31,8 @@ export class CallEventHandler { calls: Map; callEventBuffer: MatrixEvent[]; candidateEventsByCall: Map>; + nextSeqByCall: Map = new Map(); + toDeviceEventBuffers: Map> = new Map(); private eventBufferPromiseChain?: Promise; @@ -80,7 +82,45 @@ export class CallEventHandler { }; private onToDeviceEvent = (event: MatrixEvent): void => { - this.callEventBuffer.push(event); + const content = event.getContent(); + + if (!content.call_id) { + this.callEventBuffer.push(event); + return; + } + + if (!this.nextSeqByCall.has(content.call_id)) { + this.nextSeqByCall.set(content.call_id, 0); + } + + if (content.seq === undefined) { + this.callEventBuffer.push(event); + return; + } + + const nextSeq = this.nextSeqByCall.get(content.call_id) || 0; + + if (content.seq !== nextSeq) { + if (!this.toDeviceEventBuffers.has(content.call_id)) { + this.toDeviceEventBuffers.set(content.call_id, []); + } + + const buffer = this.toDeviceEventBuffers.get(content.call_id); + const index = buffer.findIndex((e) => e.getContent().seq > content.seq); + buffer.splice(index, 0, event); + } else { + const callId = content.call_id; + this.callEventBuffer.push(event); + this.nextSeqByCall.set(callId, content.seq + 1); + + const buffer = this.toDeviceEventBuffers.get(callId); + + while (buffer.length > 0 && buffer[0].getContent().seq === content.seq + 1) { + const nextEvent = buffer.pop(); + this.callEventBuffer.push(nextEvent); + this.nextSeqByCall.set(callId, nextEvent.getContent().seq + 1); + } + } }; private async evaluateEventBuffer(eventBuffer: MatrixEvent[]) { @@ -122,6 +162,8 @@ export class CallEventHandler { } private async handleCallEvent(event: MatrixEvent) { + this.client.emit("received_voip_event", event); + const content = event.getContent(); const callRoomId = ( event.getRoomId() || From 353d6bab47ab928aab089e897f5475942fcfa0ac Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 18 Feb 2022 11:35:56 -0800 Subject: [PATCH 172/291] Fix and add a test for toDevice ordering --- spec/unit/webrtc/callEventHandler.spec.ts | 83 +++++++++++++++++++++++ src/webrtc/callEventHandler.ts | 13 +++- 2 files changed, 93 insertions(+), 3 deletions(-) create mode 100644 spec/unit/webrtc/callEventHandler.spec.ts diff --git a/spec/unit/webrtc/callEventHandler.spec.ts b/spec/unit/webrtc/callEventHandler.spec.ts new file mode 100644 index 00000000000..f1e9bcbaae3 --- /dev/null +++ b/spec/unit/webrtc/callEventHandler.spec.ts @@ -0,0 +1,83 @@ +import { TestClient } from '../../TestClient'; +import { CallEventHandler } from '../../../src/webrtc/callEventHandler'; +import { MatrixEvent } from '../../../src/models/event'; +import { EventType } from '../../../src/@types/event'; + +describe('CallEventHandler', function() { + let client; + + beforeEach(function() { + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); + }); + + afterEach(function() { + client.stop(); + }); + + it('should enforce inbound toDevice message ordering', async function() { + const callEventHandler = new CallEventHandler(client); + + const event1 = new MatrixEvent({ + type: EventType.CallInvite, + content: { + call_id: "123", + seq: 0, + }, + }); + callEventHandler["onToDeviceEvent"](event1); + + expect(callEventHandler.callEventBuffer.length).toBe(1); + expect(callEventHandler.callEventBuffer[0]).toBe(event1); + + const event2 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 1, + }, + }); + callEventHandler["onToDeviceEvent"](event2); + + expect(callEventHandler.callEventBuffer.length).toBe(2); + expect(callEventHandler.callEventBuffer[1]).toBe(event2); + + const event3 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 3, + }, + }); + callEventHandler["onToDeviceEvent"](event3); + + expect(callEventHandler.callEventBuffer.length).toBe(2); + expect(callEventHandler.nextSeqByCall.get("123")).toBe(2); + expect(callEventHandler.toDeviceEventBuffers.get("123").length).toBe(1); + + const event4 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 4, + }, + }); + callEventHandler["onToDeviceEvent"](event4); + + expect(callEventHandler.callEventBuffer.length).toBe(2); + expect(callEventHandler.nextSeqByCall.get("123")).toBe(2); + expect(callEventHandler.toDeviceEventBuffers.get("123").length).toBe(2); + + const event5 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 2, + }, + }); + callEventHandler["onToDeviceEvent"](event5); + + expect(callEventHandler.callEventBuffer.length).toBe(5); + expect(callEventHandler.nextSeqByCall.get("123")).toBe(5); + expect(callEventHandler.toDeviceEventBuffers.get("123").length).toBe(0); + }); +}); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index ea454a4bd97..6c2c126bb7e 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -107,7 +107,12 @@ export class CallEventHandler { const buffer = this.toDeviceEventBuffers.get(content.call_id); const index = buffer.findIndex((e) => e.getContent().seq > content.seq); - buffer.splice(index, 0, event); + + if (index === -1) { + buffer.push(event); + } else { + buffer.splice(index, 0, event); + } } else { const callId = content.call_id; this.callEventBuffer.push(event); @@ -115,10 +120,12 @@ export class CallEventHandler { const buffer = this.toDeviceEventBuffers.get(callId); - while (buffer.length > 0 && buffer[0].getContent().seq === content.seq + 1) { - const nextEvent = buffer.pop(); + let nextEvent = buffer && buffer.shift(); + + while (nextEvent && nextEvent.getContent().seq === this.nextSeqByCall.get(callId)) { this.callEventBuffer.push(nextEvent); this.nextSeqByCall.set(callId, nextEvent.getContent().seq + 1); + nextEvent = buffer.shift(); } } }; From 7a249e3ef58df4578acb26473c71f1bfb1cf90ec Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 18 Feb 2022 14:35:09 -0800 Subject: [PATCH 173/291] Switch media devices on disconnect --- src/client.ts | 2 ++ src/webrtc/mediaHandler.ts | 28 ++++++++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/src/client.ts b/src/client.ts index 917b4215ccc..6fd60cc3b3c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1028,6 +1028,8 @@ export class MatrixClient extends EventEmitter { this.groupCallEventHandler?.stop(); this.callEventHandler = null; + this.mediaHandler.stop(); + global.clearInterval(this.checkTurnServersIntervalID); if (this.clientWellKnownIntervalID !== undefined) { global.clearInterval(this.clientWellKnownIntervalID); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index ed522020909..a49b48e2b1d 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -36,8 +36,32 @@ export class MediaHandler extends EventEmitter { constructor(private client: MatrixClient) { super(); + + navigator.mediaDevices.addEventListener("devicechange", this.onDeviceChange); } + private onDeviceChange = async (e: Event) => { + const mediaDevices = await navigator.mediaDevices.enumerateDevices(); + + logger.log("Connected media devices changed"); + + const audioInputDevices = mediaDevices.filter((device) => device.kind === "audioinput"); + const videoInputDevices = mediaDevices.filter((device) => device.kind === "audioinput"); + + const audioDeviceConnected = this.audioInput && + audioInputDevices.some((device) => device.deviceId === this.audioInput); + const videoDeviceConnected = this.videoInput && + videoInputDevices.some((device) => device.deviceId === this.videoInput); + + if (!audioDeviceConnected && audioInputDevices.length > 0) { + this.setAudioInput(audioInputDevices[0].deviceId); + } + + if (!videoDeviceConnected && videoInputDevices.length > 0) { + this.setVideoInput(videoInputDevices[0].deviceId); + } + }; + /** * Set an audio input device to use for MatrixCalls * @param {string} deviceId the identifier for the device @@ -299,4 +323,8 @@ export class MediaHandler extends EventEmitter { }; } } + + public stop() { + navigator.mediaDevices.removeEventListener("devicechange", this.onDeviceChange); + } } From 0ef6c2e35fe0962ad9c081c8ea64bd5071fc3000 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 18 Feb 2022 17:47:01 -0800 Subject: [PATCH 174/291] Add callId to all logs --- src/webrtc/call.ts | 159 ++++++++++++++++++++++++++------------------- 1 file changed, 91 insertions(+), 68 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bb00a8c9ccc..6048153972f 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -533,7 +533,8 @@ export class MatrixCall extends EventEmitter { const videoMuted = this.remoteSDPStreamMetadata[stream.id].video_muted; if (!purpose) { - logger.warn(`Ignoring stream with id ${stream.id} because we didn't get any metadata about it`); + logger.warn(`Call ${this.callId} Ignoring stream with id ${ + stream.id} because we didn't get any metadata about it`); return; } @@ -555,7 +556,8 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}", purpose=${purpose})`); + logger.info(`Call ${this.callId} Pushed remote stream (id="${ + stream.id}", active="${stream.active}", purpose=${purpose})`); } /** @@ -572,7 +574,8 @@ export class MatrixCall extends EventEmitter { // status of streams change - Dave // If we already have a stream, check this stream has the same id if (oldRemoteStream && stream.id !== oldRemoteStream.id) { - logger.warn(`Ignoring new stream ID ${stream.id}: we already have stream ID ${oldRemoteStream.id}`); + logger.warn(`Call ${this.callId} Ignoring new stream ID ${ + stream.id}: we already have stream ID ${oldRemoteStream.id}`); return; } @@ -594,7 +597,7 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.FeedsChanged, this.feeds); } - logger.info(`Pushed remote stream (id="${stream.id}", active="${stream.active}")`); + logger.info(`Call ${this.callId} Pushed remote stream (id="${stream.id}", active="${stream.active}")`); } private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { @@ -646,6 +649,7 @@ export class MatrixCall extends EventEmitter { for (const track of callFeed.stream.getTracks()) { logger.info( + `Call ${this.callId} ` + `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + @@ -658,6 +662,7 @@ export class MatrixCall extends EventEmitter { } logger.info( + `Call ${this.callId} ` + `Pushed local stream `+ `(id="${callFeed.stream.id}", `+ `active="${callFeed.stream.active}", `+ @@ -704,7 +709,7 @@ export class MatrixCall extends EventEmitter { private deleteFeedByStream(stream: MediaStream): void { const feed = this.getFeedByStreamId(stream.id); if (!feed) { - logger.warn(`Didn't find the feed with stream id ${stream.id} to delete`); + logger.warn(`Call ${this.callId} Didn't find the feed with stream id ${stream.id} to delete`); return; } this.deleteFeed(feed); @@ -751,14 +756,15 @@ export class MatrixCall extends EventEmitter { // poll and keep the credentials valid so this should be instant. const haveTurnCreds = await this.client.checkTurnServers(); if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + logger.warn(`Call ${this.callId} Failed to get TURN credentials! Proceeding with call anyway...`); } const sdpStreamMetadata = invite[SDPStreamMetadataKey]; if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - logger.debug("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + logger.debug(`Call ${ + this.callId} did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } this.peerConn = this.createPeerConnection(); @@ -770,7 +776,7 @@ export class MatrixCall extends EventEmitter { await this.peerConn.setRemoteDescription(invite.offer); await this.addBufferedIceCandidates(); } catch (e) { - logger.debug("Failed to set remote description", e); + logger.debug(`Call ${this.callId} failed to set remote description`, e); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } @@ -781,7 +787,7 @@ export class MatrixCall extends EventEmitter { // add streams until media started arriving on them. Testing latest firefox // (81 at time of writing), this is no longer a problem, so let's do it the correct way. if (!remoteStream || remoteStream.getTracks().length === 0) { - logger.error("No remote stream or no tracks after setting remote description!"); + logger.error(`Call ${this.callId} no remote stream or no tracks after setting remote description!`); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } @@ -791,7 +797,7 @@ export class MatrixCall extends EventEmitter { if (event.getLocalAge()) { setTimeout(() => { if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); + logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); this.stopAllMedia(); @@ -820,7 +826,8 @@ export class MatrixCall extends EventEmitter { ): boolean { if (wantedValue && !valueOfTheOtherSide) { // TODO: Figure out how to do this - logger.warn(`Unable to answer with ${type} because the other side isn't sending it either.`); + logger.warn(`Call ${this.callId} Unable to answer with ${ + type} because the other side isn't sending it either.`); return false; } else if ( !utils.isNullOrUndefined(wantedValue) && @@ -828,8 +835,9 @@ export class MatrixCall extends EventEmitter { !this.opponentSupportsSDPStreamMetadata() ) { logger.warn( - `Unable to answer with ${type}=${wantedValue} because the other side doesn't support it. ` + - `Answering with ${type}=${valueOfTheOtherSide}.`, + `Call ${this.callId} Unable to answer with ${type}=${ + wantedValue} because the other side doesn't support it. Answering with ${ + type}=${valueOfTheOtherSide}.`, ); return valueOfTheOtherSide; } @@ -877,7 +885,7 @@ export class MatrixCall extends EventEmitter { } catch (e) { if (answerWithVideo) { // Try to answer without video - logger.warn("Failed to getUserMedia(), trying to getUserMedia() without video"); + logger.warn(`Call ${this.callId} Failed to getUserMedia(), trying to getUserMedia() without video`); this.setState(prevState); this.waitForLocalAVStream = false; await this.answer(answerWithAudio, false); @@ -905,14 +913,15 @@ export class MatrixCall extends EventEmitter { * @param {MatrixCall} newCall The new call. */ public replacedBy(newCall: MatrixCall): void { + logger.debug(`Call ${this.callId} replaced by ${newCall.callId}`); if (this.state === CallState.WaitLocalMedia) { - logger.debug("Telling new call to wait for local media"); + logger.debug(`Call ${this.callId} telling new call ${newCall.callId} to wait for local media`); newCall.waitForLocalAVStream = true; } else if ([CallState.CreateOffer, CallState.InviteSent].includes(this.state)) { if (newCall.direction === CallDirection.Outbound) { newCall.queueGotCallFeedsForAnswer([]); } else { - logger.debug("Handing local stream to new call"); + logger.debug(`Call ${this.callId} handing local stream to new call ${newCall.callId}`); newCall.queueGotCallFeedsForAnswer(this.getLocalFeeds().map(feed => feed.clone())); } } @@ -953,7 +962,8 @@ export class MatrixCall extends EventEmitter { if (this.opponentVersion < 1) { logger.info( - `Opponent version is less than 1 (${this.opponentVersion}): sending hangup instead of reject`, + `Call ${this.callId} Opponent version is less than 1 (${ + this.opponentVersion}): sending hangup instead of reject`, ); this.hangup(CallErrorCode.UserHangup, true); return; @@ -979,7 +989,7 @@ export class MatrixCall extends EventEmitter { try { const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack; const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack; - logger.debug(`Upgrading call: audio?=${upgradeAudio} video?=${upgradeVideo}`); + logger.debug(`Upgrading call ${this.callId}: audio?=${upgradeAudio} video?=${upgradeVideo}`); const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo); if (upgradeAudio && upgradeVideo) { @@ -1001,7 +1011,7 @@ export class MatrixCall extends EventEmitter { this.peerConn.addTrack(videoTrack, this.localUsermediaStream); } } catch (error) { - logger.error("Failed to upgrade the call", error); + logger.error(`Call ${this.callId} Failed to upgrade the call`, error); this.emit(CallEvent.Error, new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error), ); @@ -1035,10 +1045,10 @@ export class MatrixCall extends EventEmitter { ): Promise { // Skip if there is nothing to do if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); + logger.warn(`Call ${this.callId} There is already a screensharing stream - there is nothing to do!`); return true; } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + logger.warn(`Call ${this.callId} There already isn't a screensharing stream - there is nothing to do!`); return false; } @@ -1047,7 +1057,7 @@ export class MatrixCall extends EventEmitter { return await this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); } - logger.debug(`Set screensharing enabled? ${enabled}`); + logger.debug(`Call ${this.callId} set screensharing enabled? ${enabled}`); if (enabled) { try { const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); @@ -1055,7 +1065,7 @@ export class MatrixCall extends EventEmitter { this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); + logger.error(`Call ${this.callId} Failed to get screen-sharing stream:`, err); return false; } } else { @@ -1078,7 +1088,7 @@ export class MatrixCall extends EventEmitter { private async setScreensharingEnabledWithoutMetadataSupport( enabled: boolean, desktopCapturerSourceId?: string, ): Promise { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + logger.debug(`Call ${this.callId} Set screensharing enabled? ${enabled} using replaceTrack()`); if (enabled) { try { const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); @@ -1096,7 +1106,7 @@ export class MatrixCall extends EventEmitter { return true; } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); + logger.error(`Call ${this.callId} Failed to get screen-sharing stream:`, err); return false; } } else { @@ -1137,6 +1147,7 @@ export class MatrixCall extends EventEmitter { try { logger.info( + `Call ${this.callId} `+ `Replacing track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + @@ -1148,6 +1159,7 @@ export class MatrixCall extends EventEmitter { newSender = oldSender; } catch (error) { logger.info( + `Call ${this.callId} `+ `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + @@ -1321,7 +1333,7 @@ export class MatrixCall extends EventEmitter { this.setState(CallState.CreateOffer); - logger.debug("gotUserMediaForInvite"); + logger.debug(`Call ${this.callId} gotUserMediaForInvite`); // Now we wait for the negotiationneeded event } @@ -1344,7 +1356,8 @@ export class MatrixCall extends EventEmitter { // We have just taken the local description from the peerConn which will // contain all the local candidates added so far, so we can discard any candidates // we had queued up because they'll be in the answer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in answer`); + logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in answer`); this.candidateSendQueue = []; try { @@ -1398,7 +1411,7 @@ export class MatrixCall extends EventEmitter { this.getRidOfRTXCodecs(); myAnswer = await this.peerConn.createAnswer(); } catch (err) { - logger.debug("Failed to create answer: ", err); + logger.debug(`Call ${this.callId} Failed to create answer: `, err); this.terminate(CallParty.Local, CallErrorCode.CreateAnswer, true); return; } @@ -1414,7 +1427,7 @@ export class MatrixCall extends EventEmitter { this.sendAnswer(); } catch (err) { - logger.debug("Error setting local description!", err); + logger.debug(`Call ${this.callId} Error setting local description!`, err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } @@ -1444,7 +1457,7 @@ export class MatrixCall extends EventEmitter { }; private onIceGatheringStateChange = (event: Event): void => { - logger.debug("ice gathering state changed to " + this.peerConn.iceGatheringState); + logger.debug(`Call ${this.callId} ice gathering state changed to ${this.peerConn.iceGatheringState}`); if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { // If we didn't get an empty-string candidate to signal the end of candidates, // create one ourselves now gathering has finished. @@ -1470,7 +1483,7 @@ export class MatrixCall extends EventEmitter { const content = ev.getContent(); const candidates = content.candidates; if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); + logger.info(`Call ${this.callId} Ignoring candidates event with no candidates!`); return; } @@ -1478,7 +1491,7 @@ export class MatrixCall extends EventEmitter { if (this.opponentPartyId === undefined) { // we haven't picked an opponent yet so save the candidates - logger.info(`Buffering ${candidates.length} candidates until we pick an opponent`); + logger.info(`Call ${this.callId} Buffering ${candidates.length} candidates until we pick an opponent`); const bufferedCandidates = this.remoteCandidateBuffer.get(fromPartyId) || []; bufferedCandidates.push(...candidates); this.remoteCandidateBuffer.set(fromPartyId, bufferedCandidates); @@ -1487,6 +1500,7 @@ export class MatrixCall extends EventEmitter { if (!this.partyIdMatches(content)) { logger.info( + `Call ${this.callId} `+ `Ignoring candidates from party ID ${content.party_id}: ` + `we have chosen party ID ${this.opponentPartyId}`, ); @@ -1512,6 +1526,7 @@ export class MatrixCall extends EventEmitter { if (this.opponentPartyId !== undefined) { logger.info( + `Call ${this.callId} ` + `Ignoring answer from party ID ${content.party_id}: ` + `we already have an answer/reject from ${this.opponentPartyId}`, ); @@ -1527,13 +1542,13 @@ export class MatrixCall extends EventEmitter { if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - logger.warn("Did not get any SDPStreamMetadata! Can not send/receive multiple streams"); + logger.warn(`Call ${this.callId} Did not get any SDPStreamMetadata! Can not send/receive multiple streams`); } try { await this.peerConn.setRemoteDescription(content.answer); } catch (e) { - logger.debug("Failed to set remote description", e); + logger.debug(`Call ${this.callId} Failed to set remote description`, e); this.terminate(CallParty.Local, CallErrorCode.SetRemoteDescription, false); return; } @@ -1549,26 +1564,28 @@ export class MatrixCall extends EventEmitter { } catch (err) { // This isn't fatal, and will just mean that if another party has raced to answer // the call, they won't know they got rejected, so we carry on & don't retry. - logger.warn("Failed to send select_answer event", err); + logger.warn(`Call ${this.callId} Failed to send select_answer event`, err); } } } public async onSelectAnswerReceived(event: MatrixEvent): Promise { if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); + logger.warn(`Call ${this.callId} Got select_answer for an outbound call: ignoring`); return; } const selectedPartyId = event.getContent().selected_party_id; if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); + logger.warn(`Call ${ + this.callId} Got nonsensical select_answer with null/undefined selected_party_id: ignoring`); return; } if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); + logger.info(`Call ${this.callId} Got select_answer for party ID ${ + selectedPartyId}: we are party ID ${this.ourPartyId}.`); // The other party has picked somebody else's answer this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); } @@ -1578,7 +1595,7 @@ export class MatrixCall extends EventEmitter { const content = event.getContent(); const description = content.description; if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); + logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); return; } // Politeness always follows the direction of the call: in a glare situation, @@ -1595,7 +1612,7 @@ export class MatrixCall extends EventEmitter { this.ignoreOffer = !polite && offerCollision; if (this.ignoreOffer) { - logger.info("Ignoring colliding negotiate event because we're impolite"); + logger.info(`Call ${this.callId} Ignoring colliding negotiate event because we're impolite`); return; } @@ -1605,7 +1622,7 @@ export class MatrixCall extends EventEmitter { if (sdpStreamMetadata) { this.updateRemoteSDPStreamMetadata(sdpStreamMetadata); } else { - logger.warn("Received negotiation event without SDPStreamMetadata!"); + logger.warn(`Call ${this.callId} Received negotiation event without SDPStreamMetadata!`); } try { @@ -1621,7 +1638,7 @@ export class MatrixCall extends EventEmitter { }); } } catch (err) { - logger.warn("Failed to complete negotiation", err); + logger.warn(`Call ${this.callId} Failed to complete negotiation`, err); } const newLocalOnHold = this.isLocalOnHold(); @@ -1690,7 +1707,7 @@ export class MatrixCall extends EventEmitter { } private async gotLocalOffer(): Promise { - logger.debug("Setting local description"); + logger.debug(`Call ${this.callId} Setting local description`); if (this.callHasEnded()) { logger.debug("Ignoring newly created offer on call ID " + this.callId + @@ -1701,7 +1718,7 @@ export class MatrixCall extends EventEmitter { try { await this.peerConn.setLocalDescription(); } catch (err) { - logger.debug("Error setting local description!", err); + logger.debug(`Call ${this.callId} Error setting local description!`, err); this.terminate(CallParty.Local, CallErrorCode.SetLocalDescription, true); return; } @@ -1741,13 +1758,14 @@ export class MatrixCall extends EventEmitter { // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. - logger.info(`Discarding ${this.candidateSendQueue.length} candidates that will be sent in offer`); + logger.info(`Call ${this.callId} Discarding ${ + this.candidateSendQueue.length} candidates that will be sent in offer`); this.candidateSendQueue = []; try { await this.sendVoipEvent(eventType, content); } catch (error) { - logger.error("Failed to send invite", error); + logger.error(`Call ${this.callId} Failed to send invite`, error); if (error.event) this.client.cancelPendingEvent(error.event); let code = CallErrorCode.SignallingFailed; @@ -1783,7 +1801,7 @@ export class MatrixCall extends EventEmitter { } private getLocalOfferFailed = (err: Error): void => { - logger.error("Failed to get local offer", err); + logger.error(`Call ${this.callId} Failed to get local offer`, err); this.emit( CallEvent.Error, @@ -1801,7 +1819,7 @@ export class MatrixCall extends EventEmitter { return; } - logger.warn("Failed to get user media - ending call", err); + logger.warn(`Failed to get user media - ending call ${this.callId}`, err); this.emit( CallEvent.Error, @@ -1856,14 +1874,14 @@ export class MatrixCall extends EventEmitter { private onTrack = (ev: RTCTrackEvent): void => { if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); + logger.warn(`Call ${this.callId} Streamless ${ev.track.kind} found: ignoring.`); return; } const stream = ev.streams[0]; this.pushRemoteFeed(stream); stream.addEventListener("removetrack", () => { - logger.log(`Removing track streamId: ${stream.id}`); + logger.log(`Call ${this.callId} removing track streamId: ${stream.id}`); this.deleteFeedByStream(stream); }); }; @@ -1914,10 +1932,11 @@ export class MatrixCall extends EventEmitter { } private onNegotiationNeeded = async (): Promise => { - logger.info("Negotiation is needed!"); + logger.info(`Call ${this.callId} Negotiation is needed!`); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); + logger.info(`Call ${ + this.callId} Opponent does not support renegotiation: ignoring negotiationneeded event`); return; } @@ -1933,7 +1952,8 @@ export class MatrixCall extends EventEmitter { // default reason is user_hangup this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); } else { - logger.info(`Ignoring message from party ID ${msg.party_id}: our partner is ${this.opponentPartyId}`); + logger.info(`Call ${this.callId} Ignoring message from party ID ${ + msg.party_id}: our partner is ${this.opponentPartyId}`); } }; @@ -1955,12 +1975,12 @@ export class MatrixCall extends EventEmitter { if (shouldTerminate) { this.terminate(CallParty.Remote, msg.reason || CallErrorCode.UserHangup, true); } else { - logger.debug(`Call is in state: ${this.state}: ignoring reject`); + logger.debug(`Call ${this.callId} is in state: ${this.state}: ignoring reject`); } }; public onAnsweredElsewhere = (msg: MCallAnswer): void => { - logger.debug("Call ID " + this.callId + " answered elsewhere"); + logger.debug("Call " + this.callId + " answered elsewhere"); this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); }; @@ -2154,7 +2174,10 @@ export class MatrixCall extends EventEmitter { } private stopAllMedia(): void { - logger.debug(!this.groupCallId ? "Stopping all media" : "Stopping all media except local feeds" ); + logger.debug( + !this.groupCallId ? + `Call ${this.callId} stopping all media` : + `Call ${this.callId} stopping all media except local feeds`); for (const feed of this.feeds) { if ( feed.isLocal() && @@ -2195,7 +2218,7 @@ export class MatrixCall extends EventEmitter { const content = { candidates: candidates, }; - logger.debug("Attempting to send " + candidates.length + " candidates"); + logger.debug(`Call ${this.callId} attempting to send " + candidates.length + " candidates`); try { await this.sendVoipEvent(EventType.CallCandidates, content); // reset our retry count if we have successfully sent our candidates @@ -2214,8 +2237,9 @@ export class MatrixCall extends EventEmitter { if (this.candidateSendTries > 5) { logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, + `Call ${this.callId} failed to send candidates on attempt ${ + this.candidateSendTries}. Giving up on this call.`, + error, ); const code = CallErrorCode.SignallingFailed; @@ -2229,7 +2253,7 @@ export class MatrixCall extends EventEmitter { const delayMs = 500 * Math.pow(2, this.candidateSendTries); ++this.candidateSendTries; - logger.debug("Failed to send candidates. Retrying in " + delayMs + "ms", error); + logger.debug(`Call ${this.callId} failed to send candidates. Retrying in ${delayMs}ms`, error); setTimeout(() => { this.sendCandidateQueue(); }, delayMs); @@ -2282,7 +2306,7 @@ export class MatrixCall extends EventEmitter { // poll and keep the credentials valid so this should be instant. const haveTurnCreds = await this.client.checkTurnServers(); if (!haveTurnCreds) { - logger.warn("Failed to get TURN credentials! Proceeding with call anyway..."); + logger.warn(`Call ${this.callId} Failed to get TURN credentials! Proceeding with call anyway...`); } // create the peer connection now so it can be gathering candidates while we get user @@ -2326,7 +2350,7 @@ export class MatrixCall extends EventEmitter { // I choo-choo-choose you const msg = ev.getContent(); - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); + logger.debug(`Call ${this.callId} choosing opponent party ID ${msg.party_id}`); this.opponentVersion = msg.version; if (this.opponentVersion === 0) { @@ -2346,7 +2370,8 @@ export class MatrixCall extends EventEmitter { private async addBufferedIceCandidates(): Promise { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId); if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + logger.info(`Call ${this.callId} Adding ${ + bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); await this.addIceCandidates(bufferedCandidates); } this.remoteCandidateBuffer = null; @@ -2358,17 +2383,15 @@ export class MatrixCall extends EventEmitter { (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - logger.debug("Ignoring remote ICE candidate with no sdpMid or sdpMLineIndex"); + logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); continue; } - logger.debug( - "Call " + this.callId + " got remote ICE " + candidate.sdpMid + " candidate: " + candidate.candidate, - ); + logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); try { await this.peerConn.addIceCandidate(candidate); } catch (err) { if (!this.ignoreOffer) { - logger.info("Failed to add remote ICE candidate", err); + logger.info(`Call ${this.callId} failed to add remote ICE candidate`, err); } } } From e270f075a456d453ccf0cb5be5acd8d989a7ef2d Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 22 Feb 2022 16:57:43 -0800 Subject: [PATCH 175/291] Fix call log --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6048153972f..4ffdebe93b0 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2218,7 +2218,7 @@ export class MatrixCall extends EventEmitter { const content = { candidates: candidates, }; - logger.debug(`Call ${this.callId} attempting to send " + candidates.length + " candidates`); + logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { await this.sendVoipEvent(EventType.CallCandidates, content); // reset our retry count if we have successfully sent our candidates From 17f5ab41917070978a12403a5c3ce08995187dbc Mon Sep 17 00:00:00 2001 From: Robert Long Date: Wed, 23 Feb 2022 15:07:13 -0800 Subject: [PATCH 176/291] Move device changes to the application. Add methods to set device ids --- src/client.ts | 2 -- src/webrtc/mediaHandler.ts | 42 ++++++++++++++------------------------ 2 files changed, 15 insertions(+), 29 deletions(-) diff --git a/src/client.ts b/src/client.ts index 6fd60cc3b3c..917b4215ccc 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1028,8 +1028,6 @@ export class MatrixClient extends EventEmitter { this.groupCallEventHandler?.stop(); this.callEventHandler = null; - this.mediaHandler.stop(); - global.clearInterval(this.checkTurnServersIntervalID); if (this.clientWellKnownIntervalID !== undefined) { global.clearInterval(this.clientWellKnownIntervalID); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index a49b48e2b1d..9d28656f8a5 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -36,31 +36,12 @@ export class MediaHandler extends EventEmitter { constructor(private client: MatrixClient) { super(); - - navigator.mediaDevices.addEventListener("devicechange", this.onDeviceChange); } - private onDeviceChange = async (e: Event) => { - const mediaDevices = await navigator.mediaDevices.enumerateDevices(); - - logger.log("Connected media devices changed"); - - const audioInputDevices = mediaDevices.filter((device) => device.kind === "audioinput"); - const videoInputDevices = mediaDevices.filter((device) => device.kind === "audioinput"); - - const audioDeviceConnected = this.audioInput && - audioInputDevices.some((device) => device.deviceId === this.audioInput); - const videoDeviceConnected = this.videoInput && - videoInputDevices.some((device) => device.deviceId === this.videoInput); - - if (!audioDeviceConnected && audioInputDevices.length > 0) { - this.setAudioInput(audioInputDevices[0].deviceId); - } - - if (!videoDeviceConnected && videoInputDevices.length > 0) { - this.setVideoInput(videoInputDevices[0].deviceId); - } - }; + public restoreMediaSettings(audioInput: string, videoInput: string) { + this.audioInput = audioInput; + this.videoInput = videoInput; + } /** * Set an audio input device to use for MatrixCalls @@ -82,6 +63,17 @@ export class MediaHandler extends EventEmitter { await this.updateLocalUsermediaStreams(); } + /** + * Set media input devices to use for MatrixCalls + * @param {string} deviceId the identifier for the device + * undefined treated as unset + */ + public async setMediaInputs(audioInput: string, videoInput: string): Promise { + this.audioInput = audioInput; + this.videoInput = videoInput; + await this.updateLocalUsermediaStreams(); + } + public async updateLocalUsermediaStreams(): Promise { const callMediaStreamParams: Map = new Map(); for (const call of this.client.callEventHandler.calls.values()) { @@ -323,8 +315,4 @@ export class MediaHandler extends EventEmitter { }; } } - - public stop() { - navigator.mediaDevices.removeEventListener("devicechange", this.onDeviceChange); - } } From c819ac634f2c1de7dfd6f6ba3d15c8f1e73554e0 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 24 Feb 2022 12:10:02 -0800 Subject: [PATCH 177/291] Fix updating local media streams --- src/webrtc/groupCall.ts | 4 ++++ src/webrtc/mediaHandler.ts | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 18c3f9ba08c..edb44b8b294 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -230,6 +230,10 @@ export class GroupCall extends EventEmitter { if (this.localCallFeed) { const oldStream = this.localCallFeed.stream; this.localCallFeed.setNewStream(stream); + const micShouldBeMuted = this.localCallFeed.isAudioMuted(); + const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); this.client.getMediaHandler().stopUserMediaStream(oldStream); } } diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 9d28656f8a5..22676776ded 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -105,6 +105,10 @@ export class MediaHandler extends EventEmitter { } for (const groupCall of this.client.groupCallEventHandler.groupCalls.values()) { + if (!groupCall.localCallFeed) { + continue; + } + const stream = await this.getUserMediaStream( true, groupCall.type === GroupCallType.Video, From e68cabc70ebe3df5c68f1632898d00e071cc3ff8 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Thu, 24 Feb 2022 12:55:08 -0800 Subject: [PATCH 178/291] Add logging for all stream creation/cloning/muting --- src/webrtc/call.ts | 6 ++++++ src/webrtc/callFeed.ts | 2 ++ src/webrtc/groupCall.ts | 9 +++++++++ src/webrtc/mediaHandler.ts | 15 +++++++++++++-- 4 files changed, 30 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4ffdebe93b0..ec9157de962 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1133,6 +1133,8 @@ export class MatrixCall extends EventEmitter { callFeed.setNewStream(stream); const micShouldBeMuted = callFeed.isAudioMuted() || this.remoteOnHold; const vidShouldBeMuted = callFeed.isVideoMuted() || this.remoteOnHold; + logger.log(`call ${this.callId} updateLocalUsermediaStream stream ${ + stream.id} micShouldBeMuted ${micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`); setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); @@ -1182,6 +1184,7 @@ export class MatrixCall extends EventEmitter { * @returns the new mute state */ public async setLocalVideoMuted(muted: boolean): Promise { + logger.log(`call ${this.callId} setLocalVideoMuted ${muted}`); if (!await this.client.getMediaHandler().hasVideoDevice()) { return this.isLocalVideoMuted(); } @@ -1214,6 +1217,7 @@ export class MatrixCall extends EventEmitter { * @returns the new mute state */ public async setMicrophoneMuted(muted: boolean): Promise { + logger.log(`call ${this.callId} setMicrophoneMuted ${muted}`); if (!await this.client.getMediaHandler().hasAudioDevice()) { return this.isMicrophoneMuted(); } @@ -1307,6 +1311,8 @@ export class MatrixCall extends EventEmitter { const micShouldBeMuted = this.localUsermediaFeed?.isAudioMuted() || this.remoteOnHold; const vidShouldBeMuted = this.localUsermediaFeed?.isVideoMuted() || this.remoteOnHold; + logger.log(`call ${this.callId} updateMuteStatus stream ${this.localUsermediaStream.id} micShouldBeMuted ${ + micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`); setTracksEnabled(this.localUsermediaStream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(this.localUsermediaStream.getVideoTracks(), !vidShouldBeMuted); } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index df754886652..5c96b50a874 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -18,6 +18,7 @@ import EventEmitter from "events"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; +import { logger } from "../logger"; const POLLING_INTERVAL = 200; // ms export const SPEAKING_THRESHOLD = -60; // dB @@ -253,6 +254,7 @@ export class CallFeed extends EventEmitter { public clone(): CallFeed { const mediaHandler = this.client.getMediaHandler(); const stream = this.stream.clone(); + logger.log(`callFeed cloning stream ${this.stream.id} newStream ${stream.id}`); if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { mediaHandler.userMediaStreams.push(stream); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index edb44b8b294..45ccb6099cf 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -191,6 +191,8 @@ export class GroupCall extends EventEmitter { } public async initLocalCallFeed(): Promise { + logger.log(`groupCall ${this.groupCallId} initLocalCallFeed`); + if (this.state !== GroupCallState.LocalCallFeedUninitialized) { throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); } @@ -232,6 +234,9 @@ export class GroupCall extends EventEmitter { this.localCallFeed.setNewStream(stream); const micShouldBeMuted = this.localCallFeed.isAudioMuted(); const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); + logger.log(`groupCall ${this.groupCallId} updateLocalUsermediaStream oldStream ${ + oldStream.id} newStream ${stream.id} micShouldBeMuted ${ + micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`); setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); this.client.getMediaHandler().stopUserMediaStream(oldStream); @@ -378,6 +383,8 @@ export class GroupCall extends EventEmitter { } if (this.localCallFeed) { + logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted stream ${ + this.localCallFeed.stream.id} muted ${muted}`); this.localCallFeed.setAudioMuted(muted); setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); } @@ -395,6 +402,8 @@ export class GroupCall extends EventEmitter { } if (this.localCallFeed) { + logger.log(`groupCall ${this.groupCallId} setLocalVideoMuted stream ${ + this.localCallFeed.stream.id} muted ${muted}`); this.localCallFeed.setVideoMuted(muted); setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); } diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 22676776ded..1673f150913 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -49,6 +49,7 @@ export class MediaHandler extends EventEmitter { * undefined treated as unset */ public async setAudioInput(deviceId: string): Promise { + logger.log(`mediaHandler setAudioInput ${deviceId}`); this.audioInput = deviceId; await this.updateLocalUsermediaStreams(); } @@ -59,6 +60,7 @@ export class MediaHandler extends EventEmitter { * undefined treated as unset */ public async setVideoInput(deviceId: string): Promise { + logger.log(`mediaHandler setVideoInput ${deviceId}`); this.videoInput = deviceId; await this.updateLocalUsermediaStreams(); } @@ -69,6 +71,7 @@ export class MediaHandler extends EventEmitter { * undefined treated as unset */ public async setMediaInputs(audioInput: string, videoInput: string): Promise { + logger.log(`mediaHandler setMediaInputs audioInput: ${audioInput} videoInput: ${videoInput}`); this.audioInput = audioInput; this.videoInput = videoInput; await this.updateLocalUsermediaStreams(); @@ -84,6 +87,7 @@ export class MediaHandler extends EventEmitter { } for (const stream of this.userMediaStreams) { + logger.log(`mediaHandler stopping all tracks for stream ${stream.id}`); for (const track of stream.getTracks()) { track.stop(); } @@ -99,6 +103,7 @@ export class MediaHandler extends EventEmitter { const { audio, video } = callMediaStreamParams.get(call.callId); + logger.log(`mediaHandler updateLocalUsermediaStreams getUserMediaStream call ${call.callId}`); const stream = await this.getUserMediaStream(audio, video); await call.updateLocalUsermediaStream(stream); @@ -109,6 +114,8 @@ export class MediaHandler extends EventEmitter { continue; } + logger.log(`mediaHandler updateLocalUsermediaStreams getUserMediaStream groupCall ${ + groupCall.groupCallId}`); const stream = await this.getUserMediaStream( true, groupCall.type === GroupCallType.Video, @@ -145,8 +152,9 @@ export class MediaHandler extends EventEmitter { (this.localUserMediaStream.getVideoTracks().length === 0 && shouldRequestVideo) ) { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); - logger.log("Getting user media with constraints", constraints); stream = await navigator.mediaDevices.getUserMedia(constraints); + logger.log(`mediaHandler getUserMediaStream constraints ${constraints} streamId ${ + stream.id} shouldRequestAudio ${shouldRequestVideo} shouldRequestVideo ${shouldRequestVideo}`); for (const track of stream.getTracks()) { const settings = track.getSettings(); @@ -161,6 +169,8 @@ export class MediaHandler extends EventEmitter { this.localUserMediaStream = stream; } else { stream = this.localUserMediaStream.clone(); + logger.log(`mediaHandler clone userMediaStream ${this.localUserMediaStream.id} new stream ${ + stream.id} shouldRequestAudio ${shouldRequestVideo} shouldRequestVideo ${shouldRequestVideo}`); if (!shouldRequestAudio) { for (const track of stream.getAudioTracks()) { @@ -186,7 +196,7 @@ export class MediaHandler extends EventEmitter { * Stops all tracks on the provided usermedia stream */ public stopUserMediaStream(mediaStream: MediaStream) { - logger.debug("Stopping usermedia stream", mediaStream.id); + logger.log(`mediaHandler stopUserMediaStream stopping stream ${mediaStream.id}`); for (const track of mediaStream.getTracks()) { track.stop(); } @@ -257,6 +267,7 @@ export class MediaHandler extends EventEmitter { */ public stopAllStreams() { for (const stream of this.userMediaStreams) { + logger.log(`mediaHandler stopAllStreams stopping stream ${stream.id}`); for (const track of stream.getTracks()) { track.stop(); } From 49994ac4fcf73e66c2bef4548a6a514177abad5d Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 25 Feb 2022 09:42:41 -0800 Subject: [PATCH 179/291] Add checks for call/groupCall ended for updateLocalUsermediaStream --- src/webrtc/call.ts | 2 +- src/webrtc/mediaHandler.ts | 12 ++++++++++-- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ec9157de962..a8aec612882 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1682,7 +1682,7 @@ export class MatrixCall extends EventEmitter { this.emit(CallEvent.AssertedIdentityChanged); } - private callHasEnded(): boolean { + public callHasEnded(): boolean { // This exists as workaround to typescript trying to be clever and erroring // when putting if (this.state === CallState.Ended) return; twice in the same // function, even though that function is async. diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 1673f150913..10c86b47c2f 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -21,7 +21,7 @@ import EventEmitter from "events"; import { GroupCallType } from "../webrtc/groupCall"; import { MatrixClient } from "../client"; import { logger } from "../logger"; -import { CallState } from "./call"; +import { GroupCallState } from ".."; export enum MediaHandlerEvent { LocalStreamsChanged = "local_streams_changed" @@ -97,7 +97,7 @@ export class MediaHandler extends EventEmitter { this.localUserMediaStream = undefined; for (const call of this.client.callEventHandler.calls.values()) { - if (call.state === CallState.Ended || !callMediaStreamParams.has(call.callId)) { + if (call.callHasEnded() || !callMediaStreamParams.has(call.callId)) { continue; } @@ -106,6 +106,10 @@ export class MediaHandler extends EventEmitter { logger.log(`mediaHandler updateLocalUsermediaStreams getUserMediaStream call ${call.callId}`); const stream = await this.getUserMediaStream(audio, video); + if (call.callHasEnded()) { + continue; + } + await call.updateLocalUsermediaStream(stream); } @@ -121,6 +125,10 @@ export class MediaHandler extends EventEmitter { groupCall.type === GroupCallType.Video, ); + if (groupCall.state === GroupCallState.Ended) { + continue; + } + await groupCall.updateLocalUsermediaStream(stream); } From 09fee4a2d980a0a13976ee56bdb8ff482200a977 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 25 Feb 2022 09:48:19 -0800 Subject: [PATCH 180/291] Allow calls to terminate properly when calling stopClient --- src/client.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/client.ts b/src/client.ts index 917b4215ccc..0885554872a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1026,7 +1026,6 @@ export class MatrixClient extends EventEmitter { this.callEventHandler?.stop(); this.groupCallEventHandler?.stop(); - this.callEventHandler = null; global.clearInterval(this.checkTurnServersIntervalID); if (this.clientWellKnownIntervalID !== undefined) { From 94c5e3757050e88ac1f60162fc75f8227682a693 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Fri, 25 Feb 2022 10:01:17 -0800 Subject: [PATCH 181/291] Fix import --- src/webrtc/mediaHandler.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 10c86b47c2f..16c91fae354 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -18,10 +18,9 @@ limitations under the License. */ import EventEmitter from "events"; -import { GroupCallType } from "../webrtc/groupCall"; +import { GroupCallType, GroupCallState } from "../webrtc/groupCall"; import { MatrixClient } from "../client"; import { logger } from "../logger"; -import { GroupCallState } from ".."; export enum MediaHandlerEvent { LocalStreamsChanged = "local_streams_changed" @@ -161,8 +160,8 @@ export class MediaHandler extends EventEmitter { ) { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); stream = await navigator.mediaDevices.getUserMedia(constraints); - logger.log(`mediaHandler getUserMediaStream constraints ${constraints} streamId ${ - stream.id} shouldRequestAudio ${shouldRequestVideo} shouldRequestVideo ${shouldRequestVideo}`); + logger.log(`mediaHandler getUserMediaStream streamId ${stream.id} shouldRequestAudio ${ + shouldRequestVideo} shouldRequestVideo ${shouldRequestVideo}`, constraints); for (const track of stream.getTracks()) { const settings = track.getSettings(); From 6e25b133123dbc49480b641937eb18102309e592 Mon Sep 17 00:00:00 2001 From: Robert Long Date: Mon, 28 Feb 2022 16:07:36 -0800 Subject: [PATCH 182/291] Send / add end-of-candidates messages --- src/webrtc/call.ts | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a8aec612882..936940d87b6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1362,9 +1362,8 @@ export class MatrixCall extends EventEmitter { // We have just taken the local description from the peerConn which will // contain all the local candidates added so far, so we can discard any candidates // we had queued up because they'll be in the answer. - logger.info(`Call ${this.callId} Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in answer`); - this.candidateSendQueue = []; + const discardCount = this.discardDuplicateCandidates(); + logger.info(`Call ${this.callId} Discarding ${discardCount} candidates that will be sent in answer`); try { await this.sendVoipEvent(EventType.CallAnswer, answerContent); @@ -1764,9 +1763,8 @@ export class MatrixCall extends EventEmitter { // Get rid of any candidates waiting to be sent: they'll be included in the local // description we just got and will send in the offer. - logger.info(`Call ${this.callId} Discarding ${ - this.candidateSendQueue.length} candidates that will be sent in offer`); - this.candidateSendQueue = []; + const discardCount = this.discardDuplicateCandidates(); + logger.info(`Call ${this.callId} Discarding ${discardCount} candidates that will be sent in offer`); try { await this.sendVoipEvent(eventType, content); @@ -2082,6 +2080,27 @@ export class MatrixCall extends EventEmitter { } } + // Discard all non-end-of-candidates messages + // Return the number of candidate messages that were discarded. + // Call this method before sending an invite or answer message + private discardDuplicateCandidates(): number { + let discardCount = 0; + const newQueue = []; + + for (let i = 0; i < this.candidateSendQueue.length; i++) { + const candidate = this.candidateSendQueue[i]; + if (candidate.candidate === "") { + newQueue.push(candidate); + } else { + discardCount++; + } + } + + this.candidateSendQueue = newQueue; + + return discardCount; + } + /* * Transfers this call to another user */ @@ -2389,10 +2408,12 @@ export class MatrixCall extends EventEmitter { (candidate.sdpMid === null || candidate.sdpMid === undefined) && (candidate.sdpMLineIndex === null || candidate.sdpMLineIndex === undefined) ) { - logger.debug(`Call ${this.callId} ignoring remote ICE candidate with no sdpMid or sdpMLineIndex`); - continue; + logger.debug(`Call ${this.callId} got remote ICE end-of-candidates`); + } else { + logger.debug(`Call ${this.callId} got remote ICE ${ + candidate.sdpMid} candidate: ${candidate.candidate}`); } - logger.debug(`Call ${this.callId} got remote ICE ${candidate.sdpMid} candidate: ${candidate.candidate}`); + try { await this.peerConn.addIceCandidate(candidate); } catch (err) { From 2a0dc39eece93efa8d0aa56556ae099f932cc6ff Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Feb 2022 09:54:14 +0000 Subject: [PATCH 183/291] Fix bug with ine-way audio after a transfer (#2193) Seems chrome at least will give you a disabled audio track if you already had another user media audio track and disabled it, so make sure our tracks are enabled when we add them. We already did this on one code path but it didn't get moved over when a new code path was added. On the plus side, we now know the reason for the ancient code that had the comment asking what it was for, so update that. --- src/webrtc/call.ts | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 936940d87b6..07b9aee9985 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -603,9 +603,11 @@ export class MatrixCall extends EventEmitter { private pushNewLocalFeed(stream: MediaStream, purpose: SDPStreamMetadataPurpose, addToPeerConnection = true): void { const userId = this.client.getUserId(); - // TODO: Find out what is going on here - // why do we enable audio (and only audio) tracks here? -- matthew + // Tracks don't always start off enabled, eg. chrome will give a disabled + // audio track if you ask for user media audio and already had one that + // you'd set to disabled (presumably because it clones them internally). setTracksEnabled(stream.getAudioTracks(), true); + setTracksEnabled(stream.getVideoTracks(), true); // We try to replace an existing feed if there already is one with the same purpose const existingFeed = this.getLocalFeeds().find((feed) => feed.purpose === purpose); @@ -654,7 +656,8 @@ export class MatrixCall extends EventEmitter { `id="${track.id}", ` + `kind="${track.kind}", ` + `streamId="${callFeed.stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + + `streamPurpose="${callFeed.purpose}", ` + + `enabled=${track.enabled}` + `) to peer connection`, ); senderArray.push(this.peerConn.addTrack(track, callFeed.stream)); @@ -2298,6 +2301,12 @@ export class MatrixCall extends EventEmitter { try { const stream = await this.client.getMediaHandler().getUserMediaStream(audio, video); + + // make sure all the tracks are enabled (same as pushNewLocalFeed - + // we probably ought to just have one code path for adding streams) + setTracksEnabled(stream.getAudioTracks(), true); + setTracksEnabled(stream.getVideoTracks(), true); + const callFeed = new CallFeed({ client: this.client, roomId: this.roomId, From ee4cbd1ec90ed9bc5bcb2b5f45b5b6a2192095ad Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 14 Jan 2022 09:43:50 +0000 Subject: [PATCH 184/291] Don't remove streams that still have tracks (#2104) If a renogotiation ends up with one track being removed, we removed the whole stream, which would cause us to lose, for example, audio rather than just video. --- src/webrtc/call.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 07b9aee9985..7ae188aabbe 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1888,8 +1888,10 @@ export class MatrixCall extends EventEmitter { const stream = ev.streams[0]; this.pushRemoteFeed(stream); stream.addEventListener("removetrack", () => { - logger.log(`Call ${this.callId} removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); + if (stream.getTracks().length === 0) { + logger.log(`Call ${this.callId} removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + } }); }; From 96ba061732bc3b5e748719072356d562e52097ba Mon Sep 17 00:00:00 2001 From: Robert Long Date: Tue, 1 Mar 2022 16:27:41 -0800 Subject: [PATCH 185/291] Fix shouldRequestAudio logging --- src/webrtc/mediaHandler.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 16c91fae354..98344f926c8 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -161,7 +161,7 @@ export class MediaHandler extends EventEmitter { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); stream = await navigator.mediaDevices.getUserMedia(constraints); logger.log(`mediaHandler getUserMediaStream streamId ${stream.id} shouldRequestAudio ${ - shouldRequestVideo} shouldRequestVideo ${shouldRequestVideo}`, constraints); + shouldRequestAudio} shouldRequestVideo ${shouldRequestVideo}`, constraints); for (const track of stream.getTracks()) { const settings = track.getSettings(); @@ -177,7 +177,7 @@ export class MediaHandler extends EventEmitter { } else { stream = this.localUserMediaStream.clone(); logger.log(`mediaHandler clone userMediaStream ${this.localUserMediaStream.id} new stream ${ - stream.id} shouldRequestAudio ${shouldRequestVideo} shouldRequestVideo ${shouldRequestVideo}`); + stream.id} shouldRequestAudio ${shouldRequestAudio} shouldRequestVideo ${shouldRequestVideo}`); if (!shouldRequestAudio) { for (const track of stream.getAudioTracks()) { From 8d9cd0fcb37188f63a938f4dd16e1f4e967cf1b6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 3 May 2022 13:14:52 +0100 Subject: [PATCH 186/291] Support for PTT group call mode (#2338) * Add PTT call mode & mute by default in PTT calls (#2311) No other parts of PTT calls implemented yet * Make the tests pass again (#2316) https://github.com/matrix-org/matrix-js-sdk/commit/3280394bf93622c096e3e260296f7f089b97846b made call use a bunch of methods that weren't mocked in the tests. * Add maximum trasmit time for PTT (#2312) on sender side by muting mic after the max transmit time has elapsed. * Don't allow user to unmute if another user is speaking (#2313) * Add maximum trasmit time for PTT on sender side by muting mic after the max transmit time has elapsed. * Don't allow user to unmute if another user is speaking Based on https://github.com/matrix-org/matrix-js-sdk/pull/2312 For https://github.com/vector-im/element-call/issues/298 * Fix createGroupCall arguments (#2325) Comma instead of a colon... --- spec/unit/webrtc/call.spec.ts | 18 +++++++++++ src/client.ts | 2 ++ src/webrtc/groupCall.ts | 50 ++++++++++++++++++++++++++++- src/webrtc/groupCallEventHandler.ts | 3 ++ 4 files changed, 72 insertions(+), 1 deletion(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 9e12db20f47..55bb3f22785 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -145,6 +145,14 @@ describe('Call', function() { client.client.mediaHandler = new MockMediaHandler; client.client.getMediaHandler = () => client.client.mediaHandler; client.httpBackend.when("GET", "/voip/turnServer").respond(200, {}); + client.client.getRoom = () => { + return { + getMember: () => { + return {}; + }, + }; + }; + call = new MatrixCall({ client: client.client, roomId: '!foo:bar', @@ -175,6 +183,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); call.peerConn.addIceCandidate = jest.fn(); @@ -192,6 +201,7 @@ describe('Call', function() { ], }; }, + getSender: () => "@test:foo", }); expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); @@ -209,6 +219,7 @@ describe('Call', function() { ], }; }, + getSender: () => "@test:foo", }); expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); @@ -236,6 +247,7 @@ describe('Call', function() { ], }; }, + getSender: () => "@test:foo", }); call.onRemoteIceCandidatesReceived({ @@ -252,6 +264,7 @@ describe('Call', function() { ], }; }, + getSender: () => "@test:foo", }); expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(0); @@ -267,6 +280,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); @@ -291,6 +305,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); const identChangedCallback = jest.fn(); @@ -308,6 +323,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); expect(identChangedCallback).toHaveBeenCalled(); @@ -347,6 +363,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); call.pushRemoteFeed(new MockMediaStream("remote_stream")); @@ -376,6 +393,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); call.setScreensharingEnabledWithoutMetadataSupport = jest.fn(); diff --git a/src/client.ts b/src/client.ts index 0885554872a..3ce67168ce9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1322,6 +1322,7 @@ export class MatrixClient extends EventEmitter { public async createGroupCall( roomId: string, type: GroupCallType, + isPtt: boolean, intent: GroupCallIntent, dataChannelsEnabled?: boolean, dataChannelOptions?: IGroupCallDataChannelOptions, @@ -1340,6 +1341,7 @@ export class MatrixClient extends EventEmitter { this, room, type, + isPtt, intent, undefined, dataChannelsEnabled, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 45ccb6099cf..32f6fa29554 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -59,6 +59,12 @@ export class GroupCallError extends Error { } } +export class OtherUserSpeakingError extends Error { + constructor() { + super("Cannot unmute: another user is speaking"); + } +} + export interface IGroupCallDataChannelOptions { ordered: boolean; maxPacketLifeTime: number; @@ -112,6 +118,7 @@ export class GroupCall extends EventEmitter { public activeSpeakerInterval = 1000; public retryCallInterval = 5000; public participantTimeout = 1000 * 15; + public pttMaxTransmitTime = 1000 * 20; public state = GroupCallState.LocalCallFeedUninitialized; public activeSpeaker?: string; // userId @@ -129,11 +136,13 @@ export class GroupCall extends EventEmitter { private retryCallLoopTimeout?: number; private retryCallCounts: Map = new Map(); private reEmitter: ReEmitter; + private transmitTimer: number | null = null; constructor( private client: MatrixClient, public room: Room, public type: GroupCallType, + public isPtt: boolean, public intent: GroupCallIntent, groupCallId?: string, private dataChannelsEnabled?: boolean, @@ -160,6 +169,7 @@ export class GroupCall extends EventEmitter { { "m.intent": this.intent, "m.type": this.type, + "io.element.ptt": this.isPtt, // TODO: Specify datachannels "dataChannelsEnabled": this.dataChannelsEnabled, "dataChannelOptions": this.dataChannelOptions, @@ -208,6 +218,11 @@ export class GroupCall extends EventEmitter { throw error; } + // start muted on ptt calls + if (this.isPtt) { + setTracksEnabled(stream.getAudioTracks(), false); + } + const userId = this.client.getUserId(); const callFeed = new CallFeed({ @@ -216,7 +231,7 @@ export class GroupCall extends EventEmitter { userId, stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0, + audioMuted: stream.getAudioTracks().length === 0 || this.isPtt, videoMuted: stream.getVideoTracks().length === 0, }); @@ -318,10 +333,20 @@ export class GroupCall extends EventEmitter { this.retryCallCounts.clear(); clearTimeout(this.retryCallLoopTimeout); + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + this.client.removeListener("Call.incoming", this.onIncomingCall); } public leave() { + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + this.dispose(); this.setState(GroupCallState.LocalCallFeedUninitialized); } @@ -329,6 +354,11 @@ export class GroupCall extends EventEmitter { public async terminate(emitStateEvent = true) { this.dispose(); + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + this.participants = []; this.client.removeListener( "RoomState.members", @@ -382,6 +412,24 @@ export class GroupCall extends EventEmitter { return false; } + // set a timer for the maximum transmit time on PTT calls + if (this.isPtt) { + // if anoher user is currently unmuted, we can't unmute + if (!muted && this.userMediaFeeds.some(f => !f.isAudioMuted())) { + throw new OtherUserSpeakingError(); + } + + // Set or clear the max transmit timer + if (!muted && this.isMicrophoneMuted()) { + this.transmitTimer = setTimeout(() => { + this.setMicrophoneMuted(true); + }, this.pttMaxTransmitTime); + } else if (muted && !this.isMicrophoneMuted()) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + } + if (this.localCallFeed) { logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted stream ${ this.localCallFeed.stream.id} muted ${muted}`); diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 99e726f1f1c..476d0acfabd 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -93,6 +93,8 @@ export class GroupCallEventHandler { return; } + const isPtt = Boolean(content["io.element.ptt"]); + let dataChannelOptions: IGroupCallDataChannelOptions | undefined; if (content?.dataChannelsEnabled && content?.dataChannelOptions) { @@ -105,6 +107,7 @@ export class GroupCallEventHandler { this.client, room, callType, + isPtt, callIntent, groupCallId, content?.dataChannelsEnabled, From 9702e8a5fa6ebb5180d7c113e1f49fde2a146dfa Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 9 May 2022 22:49:32 +0100 Subject: [PATCH 187/291] Remove test 'fix' as I can't work out why it was needed, so I can't justify keeping it in the group calls merge. It should be PRed to develop separately if needed. --- spec/unit/crypto/algorithms/megolm.spec.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/spec/unit/crypto/algorithms/megolm.spec.js b/spec/unit/crypto/algorithms/megolm.spec.js index 19fc2871321..dd846403f7a 100644 --- a/spec/unit/crypto/algorithms/megolm.spec.js +++ b/spec/unit/crypto/algorithms/megolm.spec.js @@ -321,12 +321,6 @@ describe("MegolmDecryption", function() { rotation_period_ms: 9999999999999, }, }); - - // Fix the mock to call the stuff we need it to - mockCrypto.encryptAndSendToDevices = Crypto.prototype.encryptAndSendToDevices; - mockCrypto.olmDevice = olmDevice; - mockCrypto.baseApis = mockBaseApis; - mockRoom = { getEncryptionTargetMembers: jest.fn().mockReturnValue( [{ userId: "@alice:home.server" }], From 85a6a552b57ba101cc18bcd087eb6859f5ca3fd4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2022 16:30:04 +0100 Subject: [PATCH 188/291] Make tests pass again Now we know what that bit in the crypto unit test was for... --- spec/unit/crypto/algorithms/megolm.spec.js | 7 +++++++ spec/unit/webrtc/call.spec.ts | 2 ++ 2 files changed, 9 insertions(+) diff --git a/spec/unit/crypto/algorithms/megolm.spec.js b/spec/unit/crypto/algorithms/megolm.spec.js index dd846403f7a..22888d1a3c6 100644 --- a/spec/unit/crypto/algorithms/megolm.spec.js +++ b/spec/unit/crypto/algorithms/megolm.spec.js @@ -321,6 +321,13 @@ describe("MegolmDecryption", function() { rotation_period_ms: 9999999999999, }, }); + + // Splice the real method onto the mock object as megolm uses this method + // on the crypto class in order to encrypt / start sessions + mockCrypto.encryptAndSendToDevices = Crypto.prototype.encryptAndSendToDevices; + mockCrypto.olmDevice = olmDevice; + mockCrypto.baseApis = mockBaseApis; + mockRoom = { getEncryptionTargetMembers: jest.fn().mockReturnValue( [{ userId: "@alice:home.server" }], diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 7308bb26289..fe7fa72ba18 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -471,6 +471,7 @@ describe('Call', function() { }, }; }, + getSender: () => "@test:foo", }); await call.updateLocalUsermediaStream( @@ -510,6 +511,7 @@ describe('Call', function() { [SDPStreamMetadataKey]: {}, }; }, + getSender: () => "@test:foo", }); await call.upgradeCall(false, true); From da615fd512ac396f130a165724b5dbbe3f9d1a06 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 10 May 2022 17:24:21 +0100 Subject: [PATCH 189/291] More setTimeout typings --- src/webrtc/groupCall.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c3e248c395c..e3a08245dd6 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -149,11 +149,11 @@ export class GroupCall extends TypedEventEmitter = new Map(); - private activeSpeakerLoopTimeout?: number; - private retryCallLoopTimeout?: number; + private activeSpeakerLoopTimeout?: ReturnType; + private retryCallLoopTimeout?: ReturnType; private retryCallCounts: Map = new Map(); private reEmitter: ReEmitter; - private transmitTimer: number | null = null; + private transmitTimer: ReturnType | null = null; constructor( private client: MatrixClient, From 38e54ae7f291b0153506de3089dd3900c7c4b17e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 11 May 2022 16:31:14 +0100 Subject: [PATCH 190/291] Remove PTT 'other user speaking' logic (#2362) This was also in Element Call, and whilst js-=sdk might be a more sensible place, EC has all the information to do it properly (this impl didn't take admin talk-over into account). --- src/webrtc/groupCall.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index e3a08245dd6..015e8b40a70 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -431,11 +431,6 @@ export class GroupCall extends TypedEventEmitter !f.isAudioMuted())) { - throw new OtherUserSpeakingError(); - } - // Set or clear the max transmit timer if (!muted && this.isMicrophoneMuted()) { this.transmitTimer = setTimeout(() => { From 87791cd39198227e69f82d2e8c34f49623a0986f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 13 May 2022 19:26:25 +0100 Subject: [PATCH 191/291] Fix races when muting/unmuting (#2370) await on the async operation so the promise we return resolves once everything's actuall complete --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 015e8b40a70..fbcc3a8cf0f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -450,7 +450,7 @@ export class GroupCall extends TypedEventEmitter Date: Wed, 18 May 2022 06:45:26 -0700 Subject: [PATCH 192/291] Add support for sending encrypted to-device events with OLM (#2322) --- src/crypto/index.ts | 8 ++++---- src/webrtc/call.ts | 25 ++++++++++++++----------- 2 files changed, 18 insertions(+), 15 deletions(-) diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 6fb10408f85..21ed2ad744c 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -3118,7 +3118,7 @@ export class Crypto extends TypedEventEmitter} userDeviceMap + * @param {object[]} userDeviceInfoArr * mapping from userId to deviceInfo * * @param {object} payload fields to include in the encrypted payload @@ -3129,20 +3129,20 @@ export class Crypto extends TypedEventEmitter[], + userDeviceInfoArr: IOlmDevice[], payload: object, ): Promise<{contentMap, deviceInfoByDeviceId}> { const contentMap = {}; const deviceInfoByDeviceId = new Map(); const promises = []; - for (let i = 0; i < userDeviceMap.length; i++) { + for (let i = 0; i < userDeviceInfoArr.length; i++) { const encryptedContent = { algorithm: olmlib.OLM_ALGORITHM, sender_key: this.olmDevice.deviceCurve25519Key, ciphertext: {}, }; - const val = userDeviceMap[i]; + const val = userDeviceInfoArr[i]; const userId = val.userId; const deviceInfo = val.deviceInfo; const deviceId = deviceInfo.deviceId; diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 6ce9d61e367..cbc6390d15e 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2016,7 +2016,7 @@ export class MatrixCall extends TypedEventEmitter { + private async sendVoipEvent(eventType: string, content: object): Promise { const realContent = Object.assign({}, content, { version: VOIP_PROTO_VERSION, call_id: this.callId, @@ -2041,17 +2041,20 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 19 May 2022 19:09:15 +0100 Subject: [PATCH 193/291] Handle other members having no e2e keys (#2383) Fetch the device info once at the start of the cal and cache it rather than fetching every time, and throw if we're supposed to be using e2e but the other end has no e2e keys. --- src/webrtc/call.ts | 23 ++++++++++++++++++++--- src/webrtc/groupCall.ts | 24 ++++++++++++++++++++---- 2 files changed, 40 insertions(+), 7 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cbc6390d15e..cfe94976d7b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -47,6 +47,7 @@ import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; +import { DeviceInfo } from '../crypto/deviceinfo'; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -343,6 +344,7 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal()); } + private async initOpponentCrypto() { + if (!this.opponentDeviceId) return; + + const userId = this.invitee || this.getOpponentMember().userId; + const deviceInfoMap = await this.client.crypto.deviceList.downloadKeys([userId], false); + this.opponentDeviceInfo = deviceInfoMap[userId][this.opponentDeviceId]; + if (this.opponentDeviceInfo === undefined) { + throw new Error(`No keys found for opponent device ${this.opponentDeviceId}!`); + } + } + /** * Generates and returns localSDPStreamMetadata * @returns {SDPStreamMetadata} localSDPStreamMetadata @@ -792,6 +805,7 @@ export class MatrixCall extends TypedEventEmitter { + public onMemberStateChanged = async (event: MatrixEvent) => { // The member events may be received for another room, which we will ignore. if (event.getRoomId() !== this.room.roomId) { return; @@ -751,8 +752,23 @@ export class GroupCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Screenshare); - // Safari can't send a MediaStream to multiple sources, so clone it - newCall.placeCallWithCallFeeds(this.getLocalFeeds().map(feed => feed.clone()), requestScreenshareFeed); + try { + // Safari can't send a MediaStream to multiple sources, so clone it + await newCall.placeCallWithCallFeeds( + this.getLocalFeeds().map(feed => feed.clone()), + requestScreenshareFeed, + ); + } catch (e) { + logger.warn(`Failed to place call to ${member.userId}!`, e); + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.PlaceCallFailed, + `Failed to place call to ${member.userId}.`, + ), + ); + return; + } if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); From 18e2052af2e9ba7af78e5885ce10cac0dbad8225 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 26 May 2022 13:06:57 +0100 Subject: [PATCH 194/291] Wait for mute event to send in PTT mode (#2401) This waits until the mute metadata update is sent to all the calls before telling the user they're unmuted, when in PTT mode (and only when starting to talk, ie. unmuting). This should help avoid situations where the signalling connection is slow enough that the unmute event takes long enough to reach the other side that you hear someone speak before they've apparently unmuted. Involves splitting out the method to send the metadata update. --- src/webrtc/call.ts | 13 +++++++++---- src/webrtc/groupCall.ts | 28 +++++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index cfe94976d7b..fe02b4528cb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1223,6 +1223,7 @@ export class MatrixCall extends TypedEventEmitter { + await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + } + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { if (this.successor) { this.successor.queueGotCallFeedsForAnswer(callFeeds); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index f30b13447d7..2110050525f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -430,6 +430,8 @@ export class GroupCall extends TypedEventEmitter c.sendMetadataUpdate())); + } catch (e) { + logger.info("Failed to send one or more metadata updates", e); + } + } + if (this.localCallFeed) { logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted stream ${ this.localCallFeed.stream.id} muted ${muted}`); this.localCallFeed.setAudioMuted(muted); + // I don't believe its actually necessary to enable these tracks: they + // are the one on the groupcall's own CallFeed and are cloned before being + // given to any of the actual calls, so these tracks don't actually go + // anywhere. Let's do it anyway to avoid confusion. setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); } for (const call of this.calls) { - await call.setMicrophoneMuted(muted); + setTracksEnabled(call.localUsermediaFeed.stream.getAudioTracks(), !muted); + } + + if (!sendUpdatesBefore) { + try { + await Promise.all(this.calls.map(c => c.sendMetadataUpdate())); + } catch (e) { + logger.info("Failed to send one or more metadata updates", e); + } } this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); From 34ef7bc64aa52d1aa78f2e779150922892e287de Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 31 May 2022 13:43:37 +0100 Subject: [PATCH 195/291] Mute disconnected peers in PTT mode (#2421) When we lose ICE connection to peers, set the status of their feeds to muted so to end their talking session. For https://github.com/vector-im/element-call/issues/331 --- src/webrtc/call.ts | 18 +++++++++++++++++- src/webrtc/groupCall.ts | 5 +++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index fe02b4528cb..8c736b252bc 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -292,6 +292,10 @@ export class MatrixCall extends TypedEventEmitter; @@ -1876,9 +1880,10 @@ export class MatrixCall extends TypedEventEmitter { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 2110050525f..0b54e3a9809 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -610,6 +610,9 @@ export class GroupCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Screenshare); From 5e766978b8cf80d943f796df1067722a6a5918a7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 8 Jun 2022 19:28:53 +0100 Subject: [PATCH 196/291] Set PTT mode on call correctly (#2445) And not always to true. This was causing audio & video to start muted sometimes on normal calls because the ICE connection state would change to 'checking', causing the feeds to be muted. --- src/webrtc/groupCall.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0b54e3a9809..5d6d0ad857c 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -611,7 +611,7 @@ export class GroupCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Screenshare); From b97b862fb66bafee542e3c0baac35d6576b3a75d Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 10 Jun 2022 12:01:40 +0100 Subject: [PATCH 197/291] Emit unknown device errors for group call participants without e2e (#2447) * Emit unknown device errors for group call participants without e2e There are a number of different cases here: there were some before when dealing with versions that didn't send deviceId. This catches all of them and makes all these cases emit the same error. * Add type --- src/webrtc/call.ts | 3 ++- src/webrtc/callEventHandler.ts | 18 +++++++++++------- src/webrtc/groupCall.ts | 31 +++++++++++++++++++------------ 3 files changed, 32 insertions(+), 20 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8c736b252bc..bcc1de17fd3 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -48,6 +48,7 @@ import { MatrixClient } from "../client"; import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; +import { GroupCallUnknownDeviceError } from './groupCall'; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -521,7 +522,7 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 10 Jun 2022 21:02:04 +0100 Subject: [PATCH 198/291] Only clone streams on Safari (#2450) Only enable the stream cloning behaviour on Safari: it was causing the audio renderer on Chrome (both desktop and Android) to hang, causing audio to fail sometimes in Element Call and other Chrome tabs (eg. YouTube) to fail to play audio. Fixes https://github.com/vector-im/element-call/issues/267 --- src/webrtc/groupCall.ts | 37 ++++++++++++++++++++++++++++++++----- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3a8f12eca76..5f192a08ec5 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -137,6 +137,33 @@ function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } +/** + * Returns a call feed for passing to a new call in the group call. The media + * This could be either return the passed feed as-is or a clone, depending on the + * platform. + * @returns CallFeed + */ +function feedForNewCallFromFeed(feed: CallFeed): CallFeed { + const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + + // Safari can't send a MediaStream to multiple sources, so we clone it, + // however cloning mediastreams on Chrome appears to cause the audio renderer + // to become unstable and hang: https://github.com/vector-im/element-call/issues/267 + // It's a bit arbitrary what we do for other browsers: I've made Safari the special + // case on a somewhat arbitrary basis. + // To retest later to see if this hack is still necessary: + // * In Safari, you should be able to have a group call with 2 other people and both + // of them see your video stream (either desktop or mobile Safari) + // * In Chrome, you should be able to enter a call and then go to youtube and play + // a video (both desktop & Android Chrome, although in Android you may have to + // open YouTube in incognito mode to avoid being redirected to the app.) + if (isSafari) { + return feed.clone(); + } + + return feed; +} + export class GroupCall extends TypedEventEmitter { // Config public activeSpeakerInterval = 1000; @@ -552,7 +579,9 @@ export class GroupCall extends TypedEventEmitter call.pushLocalFeed(this.localScreenshareFeed.clone()))); + await Promise.all(this.calls.map(call => call.pushLocalFeed( + feedForNewCallFromFeed(this.localScreenshareFeed), + ))); await this.sendMemberStateEvent(); @@ -626,8 +655,7 @@ export class GroupCall extends TypedEventEmitter feed.clone())); + newCall.answerWithCallFeeds(this.getLocalFeeds().map((feed) => feedForNewCallFromFeed(feed))); }; /** @@ -787,9 +815,8 @@ export class GroupCall extends TypedEventEmitter feed.purpose === SDPStreamMetadataPurpose.Screenshare); try { - // Safari can't send a MediaStream to multiple sources, so clone it await newCall.placeCallWithCallFeeds( - this.getLocalFeeds().map(feed => feed.clone()), + this.getLocalFeeds().map(feed => feedForNewCallFromFeed(feed)), requestScreenshareFeed, ); } catch (e) { From 04d0d61a0ebce6abe62f8ae397ce79a742f828b6 Mon Sep 17 00:00:00 2001 From: Travis Ralston Date: Mon, 13 Jun 2022 12:23:50 -0600 Subject: [PATCH 199/291] Change CODEOWNERS for element-call feature branch (#2457) To reduce review requests going to the "wrong" team. The team has been mirrored from the vector-im side. --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2c068fff330..dd7aeb18c10 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @matrix-org/element-web +* @matrix-org/element-call-reviewers From 9192b876d244c97417570d10bf2d4eda26eb95f6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 13 Jun 2022 20:46:34 +0100 Subject: [PATCH 200/291] Disable playback of audio for muted users (#2456) * Disable playback of audio for muted users As hopefully explained by the comment * forEach instead of map --- src/webrtc/call.ts | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index bcc1de17fd3..b04d717e90a 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1696,9 +1696,17 @@ export class MatrixCall extends TypedEventEmitter t.enabled = metadata ? !metadata.audio_muted : true); + } } } From 70449ea00331391c15afef8813ca3d63e9a50fde Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 Jun 2022 11:31:13 -0400 Subject: [PATCH 201/291] Expire call member state events after 1 hour (#2446) * Expire call member state events after 1 hour * Fix lints * Avoid a possible race --- src/webrtc/groupCall.ts | 117 ++++++++++++++++++++++++---------------- 1 file changed, 71 insertions(+), 46 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 5f192a08ec5..0b12f97f9e5 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -12,7 +12,6 @@ import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; -import { RoomStateEvent } from "../matrix"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; export enum GroupCallIntent { @@ -115,6 +114,7 @@ export interface IGroupCallRoomMemberCallState { export interface IGroupCallRoomMemberState { "m.calls": IGroupCallRoomMemberCallState[]; + "m.expires_ts": number; } export enum GroupCallState { @@ -133,6 +133,16 @@ interface ICallHandlers { onCallReplaced: (newCall: MatrixCall) => void; } +const CALL_MEMBER_STATE_TIMEOUT = 1000 * 60 * 60; // 1 hour + +const callMemberStateIsExpired = (event: MatrixEvent): boolean => { + const now = Date.now(); + const content = event?.getContent() ?? {}; + const expiresAt = typeof content["m.expires_ts"] === "number" ? content["m.expires_ts"] : -Infinity; + // The event is expired if the expiration date has passed, or if it's unreasonably far in the future + return expiresAt <= now || expiresAt > now + CALL_MEMBER_STATE_TIMEOUT * 5 / 4; +}; + function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } @@ -188,6 +198,8 @@ export class GroupCall extends TypedEventEmitter = new Map(); private reEmitter: ReEmitter; private transmitTimer: ReturnType | null = null; + private memberStateExpirationTimers: Map> = new Map(); + private resendMemberStateTimer: ReturnType | null = null; constructor( private client: MatrixClient, @@ -203,10 +215,7 @@ export class GroupCall extends TypedEventEmitter { - const deviceId = this.client.getDeviceId(); + private getMemberStateEvents(): MatrixEvent[]; + private getMemberStateEvents(userId: string): MatrixEvent | null; + private getMemberStateEvents(userId?: string): MatrixEvent[] | MatrixEvent | null { + if (userId != null) { + const event = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); + return callMemberStateIsExpired(event) ? null : event; + } else { + return this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix) + .filter(event => !callMemberStateIsExpired(event)); + } + } - return this.updateMemberCallState({ + private async sendMemberStateEvent(): Promise { + const send = () => this.updateMemberCallState({ "m.call_id": this.groupCallId, "m.devices": [ { - "device_id": deviceId, + "device_id": this.client.getDeviceId(), "session_id": this.client.getSessionId(), "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose, @@ -679,23 +690,34 @@ export class GroupCall extends TypedEventEmitter { + logger.log("Resending call member state"); + await send(); + }, CALL_MEMBER_STATE_TIMEOUT * 3 / 4); + + return res; } - private removeMemberStateEvent(): Promise { - return this.updateMemberCallState(undefined); + private async removeMemberStateEvent(): Promise { + clearInterval(this.resendMemberStateTimer); + this.resendMemberStateTimer = null; + return await this.updateMemberCallState(undefined); } private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { const localUserId = this.client.getUserId(); - const currentStateEvent = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, localUserId); - const memberStateEvent = currentStateEvent?.getContent(); + const memberState = this.getMemberStateEvents(localUserId)?.getContent(); let calls: IGroupCallRoomMemberCallState[] = []; // Sanitize existing member state event - if (memberStateEvent && Array.isArray(memberStateEvent["m.calls"])) { - calls = memberStateEvent["m.calls"].filter((call) => !!call); + if (memberState && Array.isArray(memberState["m.calls"])) { + calls = memberState["m.calls"].filter((call) => !!call); } const existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); @@ -712,6 +734,7 @@ export class GroupCall extends TypedEventEmitter { // The member events may be received for another room, which we will ignore. - if (event.getRoomId() !== this.room.roomId) { - return; - } + if (event.getRoomId() !== this.room.roomId) return; const member = this.room.getMember(event.getStateKey()); + if (!member) return; - if (!member) { - return; - } - - let callsState = event.getContent()["m.calls"]; + const ignore = () => { + this.removeParticipant(member); + clearTimeout(this.memberStateExpirationTimers.get(member.userId)); + this.memberStateExpirationTimers.delete(member.userId); + }; - if (Array.isArray(callsState)) { - callsState = callsState.filter((call) => !!call); - } + const content = event.getContent(); + const callsState = !callMemberStateIsExpired(event) && Array.isArray(content["m.calls"]) + ? content["m.calls"].filter((call) => call) + : []; // Ignore expired device data - if (!Array.isArray(callsState) || callsState.length === 0) { - logger.warn(`Ignoring member state from ${member.userId} member not in any calls.`); - this.removeParticipant(member); + if (callsState.length === 0) { + logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); + ignore(); return; } // Currently we only support a single call per room. So grab the first call. const callState = callsState[0]; - const callId = callState["m.call_id"]; if (!callId) { logger.warn(`Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`); - this.removeParticipant(member); + ignore(); return; } if (callId !== this.groupCallId) { logger.warn(`Call id ${callId} does not match group call id ${this.groupCallId}, ignoring.`); - this.removeParticipant(member); + ignore(); return; } this.addParticipant(member); + clearTimeout(this.memberStateExpirationTimers.get(member.userId)); + this.memberStateExpirationTimers.set(member.userId, setTimeout(() => { + logger.warn(`Call member state for ${member.userId} has expired`); + this.removeParticipant(member); + }, content["m.expires_ts"] - Date.now())); + // Don't process your own member. const localUserId = this.client.getUserId(); @@ -847,7 +875,7 @@ export class GroupCall extends TypedEventEmitter { - const roomState = this.room.currentState; - const memberStateEvents = roomState.getStateEvents(EventType.GroupCallMemberPrefix); - - for (const event of memberStateEvents) { + for (const event of this.getMemberStateEvents()) { const memberId = event.getStateKey(); const existingCall = this.calls.find((call) => getCallUserId(call) === memberId); const retryCallCount = this.retryCallCounts.get(memberId) || 0; From df9ffdc4081b75faf45df70bfb364eccf80ee2d1 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 Jun 2022 12:55:37 -0400 Subject: [PATCH 202/291] Don't ignore call member events with a distant future expiration date (#2466) to match updates to MSC3401 --- src/webrtc/groupCall.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 0b12f97f9e5..519b2909524 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -139,8 +139,7 @@ const callMemberStateIsExpired = (event: MatrixEvent): boolean => { const now = Date.now(); const content = event?.getContent() ?? {}; const expiresAt = typeof content["m.expires_ts"] === "number" ? content["m.expires_ts"] : -Infinity; - // The event is expired if the expiration date has passed, or if it's unreasonably far in the future - return expiresAt <= now || expiresAt > now + CALL_MEMBER_STATE_TIMEOUT * 5 / 4; + return expiresAt <= now; }; function getCallUserId(call: MatrixCall): string | null { From 5b4263bf550cc3882aba2cf2c45d26260e911c6a Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 Jun 2022 13:51:32 -0400 Subject: [PATCH 203/291] Don't block muting on determining whether the device exists (#2461) * Don't block muting on determining whether the device exists * Add comments --- src/webrtc/groupCall.ts | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 519b2909524..a9794259f1b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -458,8 +458,16 @@ export class GroupCall extends TypedEventEmitter} Whether muting/unmuting was successful + */ + public async setMicrophoneMuted(muted: boolean): Promise { + // hasAudioDevice can block indefinitely if the window has lost focus, + // and it doesn't make much sense to keep a device from being muted, so + // we always allow muted = true changes to go through + if (!muted && !await this.client.getMediaHandler().hasAudioDevice()) { return false; } @@ -514,10 +522,19 @@ export class GroupCall extends TypedEventEmitter} Whether muting/unmuting was successful + */ + public async setLocalVideoMuted(muted: boolean): Promise { + // hasAudioDevice can block indefinitely if the window has lost focus, + // and it doesn't make much sense to keep a device from being muted, so + // we always allow muted = true changes to go through + if (!muted && !await this.client.getMediaHandler().hasVideoDevice()) { return false; } @@ -533,6 +550,7 @@ export class GroupCall extends TypedEventEmitter Date: Wed, 29 Jun 2022 12:38:48 +0100 Subject: [PATCH 204/291] Fix call.collectCallStats() (#2480) Regressed by https://github.com/matrix-org/matrix-js-sdk/pull/2352 (you can just use RTCStatsReport as an iterator directly (which was was what that code was doing before) which uses entries( which gives you key/value pairs, but using forEach gives you just the value. --- src/webrtc/call.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 438a659008b..2539eaaa291 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -775,9 +775,9 @@ export class MatrixCall extends TypedEventEmitter { - stats.push(item[1]); - }); + for (const item of statsReport.values()) { + stats.push(item); + } return stats; } From f55385473013ff7e430ac0e586cf616fe13a9b24 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 29 Jun 2022 18:47:01 +0100 Subject: [PATCH 205/291] Remove the feature to disable audio from muted members (#2479) At the moment it looks like its more valuable to get the audio from people even if they're not actually shown as speaking. We can always re-introduce it later. --- src/webrtc/call.ts | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2539eaaa291..ce318dde6ce 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1701,12 +1701,6 @@ export class MatrixCall extends TypedEventEmitter t.enabled = metadata ? !metadata.audio_muted : true); - } } } From e7493fd417fe93f7e944689f40b121f739a8523c Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 29 Jun 2022 16:06:17 -0400 Subject: [PATCH 206/291] Enable DTX on audio tracks in calls (#2482) This greatly reduces the amount of bandwidth used when transmitting silence. --- package.json | 2 ++ src/webrtc/call.ts | 64 +++++++++++++++++++++++++++++++++++++++++----- yarn.lock | 10 ++++++++ 3 files changed, 69 insertions(+), 7 deletions(-) diff --git a/package.json b/package.json index 0cd570a1459..034a3b599f3 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ ], "dependencies": { "@babel/runtime": "^7.12.5", + "@types/sdp-transform": "^2.4.5", "another-json": "^0.2.0", "browser-request": "^0.3.3", "bs58": "^4.0.1", @@ -63,6 +64,7 @@ "p-retry": "^4.5.0", "qs": "^6.9.6", "request": "^2.88.2", + "sdp-transform": "^2.14.1", "unhomoglyph": "^1.0.6" }, "devDependencies": { diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ce318dde6ce..0ae2aa873ce 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -22,6 +22,8 @@ limitations under the License. * @module webrtc/call */ +import { parse as parseSdp, write as writeSdp } from "sdp-transform"; + import { logger } from '../logger'; import * as utils from '../utils'; import { MatrixEvent } from '../models/event'; @@ -171,6 +173,11 @@ export enum CallErrorCode { */ CreateAnswer = 'create_answer', + /** + * An offer could not be created + */ + CreateOffer = 'create_offer', + /** * Error code used when we fail to send the answer * for some reason other than there being unknown devices @@ -1438,6 +1445,31 @@ export class MatrixCall extends TypedEventEmitter { + if (media.type === "audio") { + media.fmtp.forEach(fmtp => fmtp.config += ";usedtx=1"); + } + }); + description.sdp = writeSdp(sdp); + } + + private async createOffer(): Promise { + const offer = await this.peerConn.createOffer(); + this.enableDtx(offer); + return offer; + } + + private async createAnswer(): Promise { + const answer = await this.peerConn.createAnswer(); + this.enableDtx(answer); + return answer; + } + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { if (this.callHasEnded()) return; @@ -1449,10 +1481,10 @@ export class MatrixCall extends TypedEventEmitter { this.makingOffer = true; try { - this.getRidOfRTXCodecs(); await this.gotLocalOffer(); } catch (e) { this.getLocalOfferFailed(e); @@ -1760,8 +1800,18 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 1 Jul 2022 11:58:00 -0400 Subject: [PATCH 207/291] Fix some MatrixCall leaks and use a shared AudioContext (#2484) * Fix some MatrixCall leaks and use a shared AudioContext These leaks, combined with the dozens of AudioContexts floating around in memory across different CallFeeds, could cause some really bad performance issues and audio crashes on Chrome. * Fully release the AudioContext in CallFeed's dispose method * Fix tests --- .eslintrc.js | 4 +++ spec/unit/webrtc/call.spec.ts | 14 ++++++++++ src/webrtc/audioContext.ts | 44 +++++++++++++++++++++++++++++ src/webrtc/call.ts | 52 ++++++++++++++++++++++++----------- src/webrtc/callFeed.ts | 15 ++++++---- 5 files changed, 108 insertions(+), 21 deletions(-) create mode 100644 src/webrtc/audioContext.ts diff --git a/.eslintrc.js b/.eslintrc.js index 1f5ce5cbd1e..6c3939bf838 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -55,6 +55,10 @@ module.exports = { // We're okay with assertion errors when we ask for them "@typescript-eslint/no-non-null-assertion": "off", + // The base rule produces false positives + "func-call-spacing": "off", + "@typescript-eslint/func-call-spacing": ["error"], + "quotes": "off", // We use a `logger` intermediary module "no-console": "error", diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 677b83a5152..9195327be0f 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -59,6 +59,17 @@ const DUMMY_SDP = ( "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" ); +class MockMediaStreamAudioSourceNode { + connect() {} +} + +class MockAudioContext { + constructor() {} + createAnalyser() { return {}; } + createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } + close() {} +} + class MockRTCPeerConnection { localDescription: RTCSessionDescription; @@ -162,6 +173,9 @@ describe('Call', function() { // @ts-ignore Mock global.document = {}; + // @ts-ignore Mock + global.AudioContext = MockAudioContext; + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); // We just stub out sendEvent: we're not interested in testing the client's // event sending code here diff --git a/src/webrtc/audioContext.ts b/src/webrtc/audioContext.ts new file mode 100644 index 00000000000..10f0dd9490b --- /dev/null +++ b/src/webrtc/audioContext.ts @@ -0,0 +1,44 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +let audioContext: AudioContext | null = null; +let refCount = 0; + +/** + * Acquires a reference to the shared AudioContext. + * It's highly recommended to reuse this AudioContext rather than creating your + * own, because multiple AudioContexts can be problematic in some browsers. + * Make sure to call releaseContext when you're done using it. + * @returns {AudioContext} The shared AudioContext + */ +export const acquireContext = (): AudioContext => { + if (audioContext === null) audioContext = new AudioContext(); + refCount++; + return audioContext; +}; + +/** + * Signals that one of the references to the shared AudioContext has been + * released, allowing the context and associated hardware resources to be + * cleaned up if nothing else is using it. + */ +export const releaseContext = () => { + refCount--; + if (refCount === 0) { + audioContext.close(); + audioContext = null; + } +}; diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 0ae2aa873ce..2c5397a76fd 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -327,6 +327,7 @@ export class MatrixCall extends TypedEventEmitter; private inviteTimeout: ReturnType; + private readonly removeTrackListeners = new Map void>(); // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold // This flag represents whether we want the other party to be on hold @@ -841,18 +842,25 @@ export class MatrixCall extends TypedEventEmitter { - if (this.state == CallState.Ringing) { - logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); - this.hangupParty = CallParty.Remote; // effectively - this.setState(CallState.Ended); - this.stopAllMedia(); - if (this.peerConn.signalingState != 'closed') { - this.peerConn.close(); - } - this.emit(CallEvent.Hangup, this); + // Time out the call if it's ringing for too long + const ringingTimer = setTimeout(() => { + logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); + this.hangupParty = CallParty.Remote; // effectively + this.setState(CallState.Ended); + this.stopAllMedia(); + if (this.peerConn.signalingState != 'closed') { + this.peerConn.close(); } + this.emit(CallEvent.Hangup, this); }, invite.lifetime - event.getLocalAge()); + + const onState = (state: CallState) => { + if (state !== CallState.Ringing) { + clearTimeout(ringingTimer); + this.off(CallEvent.State, onState); + } + }; + this.on(CallEvent.State, onState); } } @@ -1986,12 +1994,19 @@ export class MatrixCall extends TypedEventEmitter { - if (stream.getTracks().length === 0) { - logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); - this.deleteFeedByStream(stream); - } - }); + + if (!this.removeTrackListeners.has(stream)) { + const onRemoveTrack = () => { + if (stream.getTracks().length === 0) { + logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + stream.removeEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.delete(stream); + } + }; + stream.addEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.set(stream, onRemoveTrack); + } }; private onDataChannel = (ev: RTCDataChannelEvent): void => { @@ -2290,6 +2305,11 @@ export class MatrixCall extends TypedEventEmitter } private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); + if (!this.hasAudioTrack) return; + if (!this.audioContext) this.audioContext = acquireContext(); this.analyser = this.audioContext.createAnalyser(); this.analyser.fftSize = 512; @@ -211,7 +210,7 @@ export class CallFeed extends TypedEventEmitter */ public measureVolumeActivity(enabled: boolean): void { if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + if (!this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; this.measuringVolumeActivity = true; this.volumeLooper(); @@ -288,5 +287,11 @@ export class CallFeed extends TypedEventEmitter public dispose(): void { clearTimeout(this.volumeLooperTimeout); + this.stream?.removeEventListener("addtrack", this.onAddTrack); + if (this.audioContext) { + this.audioContext = null; + this.analyser = null; + releaseContext(); + } } } From ca5ac79927bff2d469ac1a2bf77a10781632771a Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 Jul 2022 17:32:17 +0100 Subject: [PATCH 208/291] Revert hack to only clone streams on safari (#2485) Reverts https://github.com/matrix-org/matrix-js-sdk/pull/2450 Looks like this wasn't really the problem (although may have made it happens faster) and the actual problem was multiple audio contexts and/or leaking peer connections as fixed in https://github.com/matrix-org/matrix-js-sdk/pull/2484 --- src/webrtc/groupCall.ts | 33 +++------------------------------ 1 file changed, 3 insertions(+), 30 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a9794259f1b..2fb7f1792ed 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -146,33 +146,6 @@ function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } -/** - * Returns a call feed for passing to a new call in the group call. The media - * This could be either return the passed feed as-is or a clone, depending on the - * platform. - * @returns CallFeed - */ -function feedForNewCallFromFeed(feed: CallFeed): CallFeed { - const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); - - // Safari can't send a MediaStream to multiple sources, so we clone it, - // however cloning mediastreams on Chrome appears to cause the audio renderer - // to become unstable and hang: https://github.com/vector-im/element-call/issues/267 - // It's a bit arbitrary what we do for other browsers: I've made Safari the special - // case on a somewhat arbitrary basis. - // To retest later to see if this hack is still necessary: - // * In Safari, you should be able to have a group call with 2 other people and both - // of them see your video stream (either desktop or mobile Safari) - // * In Chrome, you should be able to enter a call and then go to youtube and play - // a video (both desktop & Android Chrome, although in Android you may have to - // open YouTube in incognito mode to avoid being redirected to the app.) - if (isSafari) { - return feed.clone(); - } - - return feed; -} - export class GroupCall extends TypedEventEmitter { // Config public activeSpeakerInterval = 1000; @@ -598,7 +571,7 @@ export class GroupCall extends TypedEventEmitter call.pushLocalFeed( - feedForNewCallFromFeed(this.localScreenshareFeed), + this.localScreenshareFeed.clone(), ))); await this.sendMemberStateEvent(); @@ -673,7 +646,7 @@ export class GroupCall extends TypedEventEmitter feedForNewCallFromFeed(feed))); + newCall.answerWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); }; /** @@ -861,7 +834,7 @@ export class GroupCall extends TypedEventEmitter feedForNewCallFromFeed(feed)), + this.getLocalFeeds().map(feed => feed.clone()), requestScreenshareFeed, ); } catch (e) { From 6ea2885796fdf2bfbe2ef88e3c1e4152e24fb37f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 1 Jul 2022 17:44:00 +0100 Subject: [PATCH 209/291] Remove empty decryption listener (#2486) * Remove empty decryption listener This listener looks like it was left over from something as it just did nothing at all. The todevice event gets put into the call event buffer which awaits on decryption for each event before processing, so it should already wait for decryption. More info: https://github.com/vector-im/element-call/issues/428 * Unused import * Unused function! --- src/webrtc/callEventHandler.ts | 18 +----------------- 1 file changed, 1 insertion(+), 17 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index fc4da0211a5..1d130739910 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { MatrixEvent, MatrixEventEvent } from '../models/event'; +import { MatrixEvent } from '../models/event'; import { logger } from '../logger'; import { CallDirection, CallErrorCode, CallState, createNewMatrixCall, MatrixCall } from './call'; import { EventType } from '../@types/event'; @@ -140,13 +140,6 @@ export class CallEventHandler { this.nextSeqByCall.set(content.call_id, 0); } - if (event.isBeingDecrypted() || event.isDecryptionFailure()) { - // add an event listener for once the event is decrypted. - event.once(MatrixEventEvent.Decrypted, async () => { - if (!this.eventIsACall(event)) return; - }); - } - if (content.seq === undefined) { this.callEventBuffer.push(event); return; @@ -184,15 +177,6 @@ export class CallEventHandler { } }; - private eventIsACall(event: MatrixEvent): boolean { - const type = event.getType(); - /** - * Unstable prefixes: - * - org.matrix.call. : MSC3086 https://github.com/matrix-org/matrix-doc/pull/3086 - */ - return type.startsWith("m.call.") || type.startsWith("org.matrix.call."); - } - private async handleCallEvent(event: MatrixEvent) { this.client.emit(ClientEvent.ReceivedVoipEvent, event); From e980c889016b48268031b22735e70b7f618e219c Mon Sep 17 00:00:00 2001 From: Robin Date: Sat, 2 Jul 2022 09:19:07 -0400 Subject: [PATCH 210/291] Don't mute the remote side immediately in PTT calls (#2487) This clause was causing all PTT calls to mute the remote side immediately upon ICE connection status changing to 'checking'. --- src/webrtc/call.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2c5397a76fd..b79b9c62157 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1971,7 +1971,7 @@ export class MatrixCall extends TypedEventEmitter Date: Tue, 5 Jul 2022 13:17:23 +0100 Subject: [PATCH 211/291] Add config option for e2e group call signalling (#2492) --- src/client.ts | 19 +++++++++++++++++++ src/webrtc/call.ts | 36 +++++++++++++++++++++++------------- 2 files changed, 42 insertions(+), 13 deletions(-) diff --git a/src/client.ts b/src/client.ts index 32e05e97260..3deed4e9262 100644 --- a/src/client.ts +++ b/src/client.ts @@ -346,6 +346,12 @@ export interface ICreateClientOpts { */ fallbackICEServerAllowed?: boolean; + /** + * If true, to-device signalling for group calls will be encrypted + * with Olm. Default: true. + */ + useE2eForGroupCall?: boolean; + cryptoCallbacks?: ICryptoCallbacks; } @@ -954,6 +960,8 @@ export class MatrixClient extends TypedEventEmitter>(); + private useE2eForGroupCall = true; + constructor(opts: IMatrixClientCreateOpts) { super(); @@ -1044,6 +1052,8 @@ export class MatrixClient extends TypedEventEmitter Date: Wed, 6 Jul 2022 22:04:41 +0100 Subject: [PATCH 212/291] Set max average bitrate on PTT calls (#2499) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Set max average bitrate on PTT calls Via SDP munging. Also makes the SDP munging a bit more generic and codec-specific (we were previously adding usedtx to any codec that had an fmtp line already, which was probably not really the intention). * Make SDP munging for codecs that don't already have fmtp lines * Use sensible typescript syntax Co-authored-by: Šimon Brandner Co-authored-by: Šimon Brandner --- src/webrtc/call.ts | 63 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 58 insertions(+), 5 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index f56d5342fdc..c1161f333a2 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -93,6 +93,14 @@ interface AssertedIdentity { displayName: string; } +// Used internally to specify modifications to codec parameters in SDP +interface CodecParams { + enableDtx?: boolean; // true to enable discontinuous transmission, false to disable, undefined to leave as-is + maxAverageBitrate?: number; // sets the max average bitrate, or undefined to leave as-is +} + +type CodecParamMods = Record; + export enum CallState { Fledgling = 'fledgling', InviteSent = 'invite_sent', @@ -261,6 +269,18 @@ export function genCallID(): string { return Date.now().toString() + randomString(16); } +function getCodecParamMods(isPtt: boolean): CodecParamMods { + const mods = { + 'opus': { + enableDtx: true, + }, + } as CodecParamMods; + + if (isPtt) mods.opus.maxAverageBitrate = 12000; + + return mods; +} + export type CallEventHandlerMap = { [CallEvent.DataChannel]: (channel: RTCDataChannel) => void; [CallEvent.FeedsChanged]: (feeds: CallFeed[]) => void; @@ -1456,12 +1476,45 @@ export class MatrixCall extends TypedEventEmitter { - if (media.type === "audio") { - media.fmtp.forEach(fmtp => fmtp.config += ";usedtx=1"); + const payloadTypeToCodecMap = new Map(); + const codecToPayloadTypeMap = new Map(); + for (const rtp of media.rtp) { + payloadTypeToCodecMap.set(rtp.payload, rtp.codec); + codecToPayloadTypeMap.set(rtp.codec, rtp.payload); + } + + for (const [codec, params] of Object.entries(mods)) { + if (!codecToPayloadTypeMap.has(codec)) { + logger.info(`Ignoring SDP modifications for ${codec} as it's not present.`); + continue; + } + + const extraconfig: string[] = []; + if (params.enableDtx !== undefined) { + extraconfig.push(`usedtx=${params.enableDtx ? '1' : '0'}`); + } + if (params.maxAverageBitrate !== undefined) { + extraconfig.push(`maxaveragebitrate=${params.maxAverageBitrate}`); + } + + let found = false; + for (const fmtp of media.fmtp) { + if (payloadTypeToCodecMap.get(fmtp.payload) === codec) { + found = true; + fmtp.config += ";" + extraconfig.join(";"); + } + } + if (!found) { + media.fmtp.push({ + payload: codecToPayloadTypeMap.get(codec), + config: extraconfig.join(";"), + }); + } } }); description.sdp = writeSdp(sdp); @@ -1469,13 +1522,13 @@ export class MatrixCall extends TypedEventEmitter { const offer = await this.peerConn.createOffer(); - this.enableDtx(offer); + this.mungeSdp(offer, getCodecParamMods(this.isPtt)); return offer; } private async createAnswer(): Promise { const answer = await this.peerConn.createAnswer(); - this.enableDtx(answer); + this.mungeSdp(answer, getCodecParamMods(this.isPtt)); return answer; } From 984dd26a138411ef73903ff4e635f2752e0829f2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 8 Jul 2022 09:45:02 +0100 Subject: [PATCH 213/291] Prevent double mute status changed events (#2502) Audio & video mute status were set in separate calls but share a mute status changed event, so you'd always get two mute status changed events emitted. We could suppress events where the mute status didn't change, but this would still get two events saying the same thing when they both changed. Instead, merge setAudioMuted & setVideoMuted into a single call that sets either or both. --- src/webrtc/call.ts | 10 ++++------ src/webrtc/callFeed.ts | 23 ++++++++++------------- src/webrtc/groupCall.ts | 6 +++--- 3 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c1161f333a2..c23133ea493 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1262,7 +1262,7 @@ export class MatrixCall extends TypedEventEmitter } /** - * Set feed's internal audio mute state - * @param muted is the feed's audio muted? - */ - public setAudioMuted(muted: boolean): void { - this.audioMuted = muted; - this.speakingVolumeSamples.fill(-Infinity); - this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); - } - - /** - * Set feed's internal video mute state + * Set one or both of feed's internal audio and video video mute state + * Either value may be null to leave it as-is * @param muted is the feed's video muted? */ - public setVideoMuted(muted: boolean): void { - this.videoMuted = muted; + public setAudioVideoMuted(audioMuted: boolean, videoMuted: boolean): void { + if (audioMuted !== null) { + if (this.audioMuted !== audioMuted) { + this.speakingVolumeSamples.fill(-Infinity); + } + this.audioMuted = audioMuted; + } + if (videoMuted !== null) this.videoMuted = videoMuted; this.emit(CallFeedEvent.MuteStateChanged, this.audioMuted, this.videoMuted); } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 2fb7f1792ed..a494447a0fc 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -460,7 +460,7 @@ export class GroupCall extends TypedEventEmitter Date: Tue, 12 Jul 2022 19:27:41 +0100 Subject: [PATCH 214/291] Merge develop into group call branch again (#2513) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Send call version `1` as a string (#2471) * test typescriptification - backup.spec (#2468) * renamed: spec/unit/crypto/crypto-utils.js -> spec/unit/crypto/crypto-utils.ts * ts fixes in crypto-utils * renamed: spec/unit/crypto/backup.spec.js -> spec/unit/crypto/backup.spec.ts * ts fixes in backup.spec * remove fit * remove debug * Prepare changelog for v19.0.0-rc.1 * v19.0.0-rc.1 * Update jest monorepo (#2476) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * Update all (#2475) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * Update dependency @types/jest to v28 (#2478) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * Fix call.collectCallStats() (#2480) Regressed by https://github.com/matrix-org/matrix-js-sdk/pull/2352 (you can just use RTCStatsReport as an iterator directly (which was was what that code was doing before) which uses entries( which gives you key/value pairs, but using forEach gives you just the value. * Go back to forEach in collectcallstats (#2481) Older typescript library doesn't know about .values() on the stats object, so it was failing in react sdk which had an older typescript. https://github.com/matrix-org/matrix-react-sdk/pull/8935 was an attempt to upgrade it but did not seem to be helping on CI, despite being fine locally. * Update babel monorepo to v7.18.6 (#2477) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * Expose KNOWN_SAFE_ROOM_VERSION (#2474) * Fix return type on funcs in matrixClient to be optionally null (#2488) * Update pull_request.yaml (#2490) * Lock file maintenance (#2491) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> * Prepare changelog for v19.0.0 * v19.0.0 * Resetting package fields for development * Improve VoIP integrations testing (#2495) * Remove MSC3244 support (#2504) * Actually store the identity server in the client when given as an option (#2503) * Actually store the identity server in the client when given as an option * Update requestRegisterEmailToken to a modern spec version too * Properly re-insert room ID in bundled thread relation messages from sync (#2505) Events returned by the `/sync` endpoint, including relations bundled with other events, may have their `room_id`s stripped out. This causes decryption errors if the IDs aren't repopulated. Fixes vector-im/element-web#22094. * Remove `setNow` from `realtime-callbacks.ts` (#2509) Signed-off-by: Šimon Brandner * Remove dead code (#2510) * Don't crash with undefined room in `processBeaconEvents()` (#2500) * Add a basic PR checklist for all PRs (#2511) It'll be mildly annoying for core developers who have to constantly remove or edit this, but it'll also serve as a good reminder to do these things. Note that signoff is not required for core developers. * Fix tests Co-authored-by: Šimon Brandner Co-authored-by: Kerry Co-authored-by: RiotRobot Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: Michael Weimann Co-authored-by: texuf Co-authored-by: Michael Telatynski <7t3chguy@gmail.com> Co-authored-by: Travis Ralston Co-authored-by: Faye Duxovni --- .github/PULL_REQUEST_TEMPLATE.md | 14 +- .github/workflows/pull_request.yaml | 1 - CHANGELOG.md | 28 + package.json | 6 +- spec/TestClient.ts | 4 + ...s => matrix-client-event-timeline.spec.ts} | 186 +- spec/integ/matrix-client-methods.spec.js | 70 +- spec/test-utils/test-utils.ts | 29 +- spec/test-utils/webrtc.ts | 157 ++ .../crypto/{backup.spec.js => backup.spec.ts} | 231 +- .../{crypto-utils.js => crypto-utils.ts} | 14 +- spec/unit/realtime-callbacks.spec.js | 10 +- spec/unit/utils.spec.ts | 12 - spec/unit/webrtc/call.spec.ts | 525 ++-- spec/unit/webrtc/callFeed.spec.ts | 61 + src/client.ts | 35 +- src/crypto/api.ts | 5 - src/crypto/verification/Error.ts | 12 - src/models/room.ts | 2 +- src/models/thread.ts | 5 +- src/realtime-callbacks.ts | 20 +- src/utils.ts | 80 - src/webrtc/call.ts | 12 +- yarn.lock | 2430 ++++++++--------- 24 files changed, 2208 insertions(+), 1741 deletions(-) rename spec/integ/{matrix-client-event-timeline.spec.js => matrix-client-event-timeline.spec.ts} (86%) create mode 100644 spec/test-utils/webrtc.ts rename spec/unit/crypto/{backup.spec.js => backup.spec.ts} (78%) rename spec/unit/crypto/{crypto-utils.js => crypto-utils.ts} (78%) create mode 100644 spec/unit/webrtc/callFeed.spec.ts diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index a71d86006a0..c0b7939a69b 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,13 @@ - + - +## Checklist - diff --git a/.github/workflows/pull_request.yaml b/.github/workflows/pull_request.yaml index 6cb9368886e..5f816811646 100644 --- a/.github/workflows/pull_request.yaml +++ b/.github/workflows/pull_request.yaml @@ -16,7 +16,6 @@ concurrency: ${{ github.workflow }}-${{ github.event.pull_request.head.ref }} jobs: changelog: name: Preview Changelog - if: github.event.action != 'synchronize' runs-on: ubuntu-latest steps: - uses: matrix-org/allchange@main diff --git a/CHANGELOG.md b/CHANGELOG.md index bf48bc37eaf..9c8e28925d5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,31 @@ +Changes in [19.0.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.0.0) (2022-07-05) +================================================================================================== + +## 🚨 BREAKING CHANGES + * Remove unused sessionStore ([\#2455](https://github.com/matrix-org/matrix-js-sdk/pull/2455)). + +## ✨ Features + * Implement MSC3827: Filtering of `/publicRooms` by room type ([\#2469](https://github.com/matrix-org/matrix-js-sdk/pull/2469)). + * expose latestLocationEvent on beacon model ([\#2467](https://github.com/matrix-org/matrix-js-sdk/pull/2467)). Contributed by @kerryarchibald. + * Live location share - add start time leniency ([\#2465](https://github.com/matrix-org/matrix-js-sdk/pull/2465)). Contributed by @kerryarchibald. + * Log real errors and not just their messages, traces are useful ([\#2464](https://github.com/matrix-org/matrix-js-sdk/pull/2464)). + * Various changes to `src/crypto` files for correctness ([\#2137](https://github.com/matrix-org/matrix-js-sdk/pull/2137)). Contributed by @ShadowJonathan. + * Update MSC3786 implementation: Check the `state_key` ([\#2429](https://github.com/matrix-org/matrix-js-sdk/pull/2429)). + * Timeline needs to refresh when we see a MSC2716 marker event ([\#2299](https://github.com/matrix-org/matrix-js-sdk/pull/2299)). Contributed by @MadLittleMods. + * Try to load keys from key backup when a message fails to decrypt ([\#2373](https://github.com/matrix-org/matrix-js-sdk/pull/2373)). Fixes vector-im/element-web#21026. Contributed by @duxovni. + +## 🐛 Bug Fixes + * Send call version `1` as a string ([\#2471](https://github.com/matrix-org/matrix-js-sdk/pull/2471)). Fixes vector-im/element-web#22629. + * Fix issue with `getEventTimeline` returning undefined for thread roots in main timeline ([\#2454](https://github.com/matrix-org/matrix-js-sdk/pull/2454)). Fixes vector-im/element-web#22539. + * Add missing `type` property on `IAuthData` ([\#2463](https://github.com/matrix-org/matrix-js-sdk/pull/2463)). + * Clearly indicate that `lastReply` on a Thread can return falsy ([\#2462](https://github.com/matrix-org/matrix-js-sdk/pull/2462)). + * Fix issues with getEventTimeline and thread roots ([\#2444](https://github.com/matrix-org/matrix-js-sdk/pull/2444)). Fixes vector-im/element-web#21613. + * Live location sharing - monitor liveness of beacons yet to start ([\#2437](https://github.com/matrix-org/matrix-js-sdk/pull/2437)). Contributed by @kerryarchibald. + * Refactor Relations to not be per-EventTimelineSet ([\#2412](https://github.com/matrix-org/matrix-js-sdk/pull/2412)). Fixes #2399 and vector-im/element-web#22298. + * Add tests for sendEvent threadId handling ([\#2435](https://github.com/matrix-org/matrix-js-sdk/pull/2435)). Fixes vector-im/element-web#22433. + * Make sure `encryptAndSendKeysToDevices` assumes devices are unique per-user. ([\#2136](https://github.com/matrix-org/matrix-js-sdk/pull/2136)). Fixes #2135. Contributed by @ShadowJonathan. + * Don't bug the user while re-checking key backups after decryption failures ([\#2430](https://github.com/matrix-org/matrix-js-sdk/pull/2430)). Fixes vector-im/element-web#22416. Contributed by @duxovni. + Changes in [18.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v18.1.0) (2022-06-07) ================================================================================================== diff --git a/package.json b/package.json index 034a3b599f3..efe0bbfe031 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "18.1.0", + "version": "19.0.0", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=12.9.0" @@ -83,7 +83,7 @@ "@matrix-org/olm": "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.8.tgz", "@types/bs58": "^4.0.1", "@types/content-type": "^1.1.5", - "@types/jest": "^27.0.0", + "@types/jest": "^28.0.0", "@types/node": "12", "@types/request": "^2.48.5", "@typescript-eslint/eslint-plugin": "^5.6.0", @@ -94,7 +94,7 @@ "better-docs": "^2.4.0-beta.9", "browserify": "^17.0.0", "docdash": "^1.2.0", - "eslint": "8.16.0", + "eslint": "8.18.0", "eslint-config-google": "^0.14.0", "eslint-plugin-import": "^2.25.4", "eslint-plugin-matrix-org": "^0.5.0", diff --git a/spec/TestClient.ts b/spec/TestClient.ts index 244a9d6e3a3..52d7eb3789d 100644 --- a/spec/TestClient.ts +++ b/spec/TestClient.ts @@ -236,4 +236,8 @@ export class TestClient { public isFallbackICEServerAllowed(): boolean { return true; } + + public getUserId(): string { + return this.userId; + } } diff --git a/spec/integ/matrix-client-event-timeline.spec.js b/spec/integ/matrix-client-event-timeline.spec.ts similarity index 86% rename from spec/integ/matrix-client-event-timeline.spec.js rename to spec/integ/matrix-client-event-timeline.spec.ts index c165a7057ed..3bde9dd6da3 100644 --- a/spec/integ/matrix-client-event-timeline.spec.js +++ b/spec/integ/matrix-client-event-timeline.spec.ts @@ -1,5 +1,21 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + import * as utils from "../test-utils/test-utils"; -import { EventTimeline, Filter, MatrixEvent } from "../../src/matrix"; +import { ClientEvent, EventTimeline, Filter, IEvent, MatrixClient, MatrixEvent, Room } from "../../src/matrix"; import { logger } from "../../src/logger"; import { TestClient } from "../TestClient"; import { Thread, THREAD_RELATION_TYPE } from "../../src/models/thread"; @@ -10,8 +26,14 @@ const accessToken = "aseukfgwef"; const roomId = "!foo:bar"; const otherUserId = "@bob:localhost"; +const withoutRoomId = (e: Partial): Partial => { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { room_id: _, ...copy } = e; + return copy; +}; + const USER_MEMBERSHIP_EVENT = utils.mkMembership({ - room: roomId, mship: "join", user: userId, name: userName, + room: roomId, mship: "join", user: userId, name: userName, event: false, }); const ROOM_NAME_EVENT = utils.mkEvent({ @@ -19,34 +41,37 @@ const ROOM_NAME_EVENT = utils.mkEvent({ content: { name: "Old room name", }, + event: false, }); const INITIAL_SYNC_DATA = { next_batch: "s_5_3", rooms: { join: { - "!foo:bar": { // roomId + [roomId]: { timeline: { events: [ utils.mkMessage({ - room: roomId, user: otherUserId, msg: "hello", + user: otherUserId, msg: "hello", event: false, }), ], prev_batch: "f_1_1", }, state: { events: [ - ROOM_NAME_EVENT, + withoutRoomId(ROOM_NAME_EVENT), utils.mkMembership({ - room: roomId, mship: "join", + mship: "join", user: otherUserId, name: "Bob", + event: false, }), - USER_MEMBERSHIP_EVENT, + withoutRoomId(USER_MEMBERSHIP_EVENT), utils.mkEvent({ - type: "m.room.create", room: roomId, user: userId, + type: "m.room.create", user: userId, content: { creator: userId, }, + event: false, }), ], }, @@ -57,16 +82,16 @@ const INITIAL_SYNC_DATA = { const EVENTS = [ utils.mkMessage({ - room: roomId, user: userId, msg: "we", + room: roomId, user: userId, msg: "we", event: false, }), utils.mkMessage({ - room: roomId, user: userId, msg: "could", + room: roomId, user: userId, msg: "could", event: false, }), utils.mkMessage({ - room: roomId, user: userId, msg: "be", + room: roomId, user: userId, msg: "be", event: false, }), utils.mkMessage({ - room: roomId, user: userId, msg: "heroes", + room: roomId, user: userId, msg: "heroes", event: false, }), ]; @@ -81,12 +106,13 @@ const THREAD_ROOT = utils.mkEvent({ unsigned: { "m.relations": { "io.element.thread": { - "latest_event": undefined, + //"latest_event": undefined, "count": 1, "current_user_participated": true, }, }, }, + event: false, }); const THREAD_REPLY = utils.mkEvent({ @@ -102,12 +128,25 @@ const THREAD_REPLY = utils.mkEvent({ event_id: THREAD_ROOT.event_id, }, }, + event: false, }); THREAD_ROOT.unsigned["m.relations"]["io.element.thread"].latest_event = THREAD_REPLY; +const SYNC_THREAD_ROOT = withoutRoomId(THREAD_ROOT); +const SYNC_THREAD_REPLY = withoutRoomId(THREAD_REPLY); +SYNC_THREAD_ROOT.unsigned = { + "m.relations": { + "io.element.thread": { + "latest_event": SYNC_THREAD_REPLY, + "count": 1, + "current_user_participated": true, + }, + }, +}; + // start the client, and wait for it to initialise -function startClient(httpBackend, client) { +function startClient(httpBackend: TestClient["httpBackend"], client: MatrixClient) { httpBackend.when("GET", "/versions").respond(200, {}); httpBackend.when("GET", "/pushrules").respond(200, {}); httpBackend.when("POST", "/filter").respond(200, { filter_id: "fid" }); @@ -116,8 +155,8 @@ function startClient(httpBackend, client) { client.startClient(); // set up a promise which will resolve once the client is initialised - const prom = new Promise((resolve) => { - client.on("sync", function(state) { + const prom = new Promise((resolve) => { + client.on(ClientEvent.Sync, function(state) { logger.log("sync", state); if (state != "SYNCING") { return; @@ -133,8 +172,8 @@ function startClient(httpBackend, client) { } describe("getEventTimeline support", function() { - let httpBackend; - let client; + let httpBackend: TestClient["httpBackend"]; + let client: MatrixClient; beforeEach(function() { const testClient = new TestClient(userId, "DEVICE", accessToken); @@ -177,7 +216,7 @@ describe("getEventTimeline support", function() { it("scrollback should be able to scroll back to before a gappy /sync", function() { // need a client with timelineSupport disabled to make this work - let room; + let room: Room; return startClient(httpBackend, client).then(function() { room = client.getRoom(roomId); @@ -189,7 +228,7 @@ describe("getEventTimeline support", function() { "!foo:bar": { timeline: { events: [ - EVENTS[0], + withoutRoomId(EVENTS[0]), ], prev_batch: "f_1_1", }, @@ -205,7 +244,7 @@ describe("getEventTimeline support", function() { "!foo:bar": { timeline: { events: [ - EVENTS[1], + withoutRoomId(EVENTS[1]), ], limited: true, prev_batch: "f_1_2", @@ -240,8 +279,8 @@ describe("getEventTimeline support", function() { }); describe("MatrixClient event timelines", function() { - let client = null; - let httpBackend = null; + let client: MatrixClient; + let httpBackend: TestClient["httpBackend"]; beforeEach(function() { const testClient = new TestClient( @@ -260,7 +299,7 @@ describe("MatrixClient event timelines", function() { afterEach(function() { httpBackend.verifyNoOutstandingExpectation(); client.stopClient(); - Thread.setServerSideSupport(false); + Thread.setServerSideSupport(false, false); }); describe("getEventTimeline", function() { @@ -308,7 +347,7 @@ describe("MatrixClient event timelines", function() { "!foo:bar": { timeline: { events: [ - EVENTS[0], + withoutRoomId(EVENTS[0]), ], prev_batch: "f_1_2", }, @@ -343,7 +382,7 @@ describe("MatrixClient event timelines", function() { "!foo:bar": { timeline: { events: [ - EVENTS[3], + withoutRoomId(EVENTS[3]), ], prev_batch: "f_1_2", }, @@ -366,7 +405,7 @@ describe("MatrixClient event timelines", function() { }); const prom = new Promise((resolve, reject) => { - client.on("sync", function() { + client.on(ClientEvent.Sync, function() { client.getEventTimeline(timelineSet, EVENTS[2].event_id, ).then(function(tl) { expect(tl.getEvents().length).toEqual(4); @@ -511,8 +550,9 @@ describe("MatrixClient event timelines", function() { }); it("should handle thread replies with server support by fetching a contiguous thread timeline", async () => { + // @ts-ignore client.clientOpts.experimentalThreadSupport = true; - Thread.setServerSideSupport(true); + Thread.setServerSideSupport(true, false); client.stopClient(); // we don't need the client to be syncing at this time const room = client.getRoom(roomId); const thread = room.createThread(THREAD_ROOT.event_id, undefined, [], false); @@ -556,8 +596,9 @@ describe("MatrixClient event timelines", function() { }); it("should return relevant timeline from non-thread timelineSet when asking for the thread root", async () => { + // @ts-ignore client.clientOpts.experimentalThreadSupport = true; - Thread.setServerSideSupport(true); + Thread.setServerSideSupport(true, false); client.stopClient(); // we don't need the client to be syncing at this time const room = client.getRoom(roomId); const threadRoot = new MatrixEvent(THREAD_ROOT); @@ -587,8 +628,9 @@ describe("MatrixClient event timelines", function() { }); it("should return undefined when event is not in the thread that the given timelineSet is representing", () => { + // @ts-ignore client.clientOpts.experimentalThreadSupport = true; - Thread.setServerSideSupport(true); + Thread.setServerSideSupport(true, false); client.stopClient(); // we don't need the client to be syncing at this time const room = client.getRoom(roomId); const threadRoot = new MatrixEvent(THREAD_ROOT); @@ -614,8 +656,9 @@ describe("MatrixClient event timelines", function() { }); it("should return undefined when event is within a thread but timelineSet is not", () => { + // @ts-ignore client.clientOpts.experimentalThreadSupport = true; - Thread.setServerSideSupport(true); + Thread.setServerSideSupport(true, false); client.stopClient(); // we don't need the client to be syncing at this time const room = client.getRoom(roomId); const timelineSet = room.getTimelineSets()[0]; @@ -639,6 +682,7 @@ describe("MatrixClient event timelines", function() { }); it("should should add lazy loading filter when requested", async () => { + // @ts-ignore client.clientOpts.lazyLoadMembers = true; client.stopClient(); // we don't need the client to be syncing at this time const room = client.getRoom(roomId); @@ -656,7 +700,7 @@ describe("MatrixClient event timelines", function() { }; }); req.check((request) => { - expect(request.opts.qs.filter).toEqual(JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER)); + expect(request.queryParams.filter).toEqual(JSON.stringify(Filter.LAZY_LOADING_MESSAGES_FILTER)); }); await Promise.all([ @@ -863,7 +907,7 @@ describe("MatrixClient event timelines", function() { "!foo:bar": { timeline: { events: [ - event, + withoutRoomId(event), ], prev_batch: "f_1_1", }, @@ -941,11 +985,10 @@ describe("MatrixClient event timelines", function() { // a state event, followed by a redaction thereof const event = utils.mkMembership({ - room: roomId, mship: "join", user: otherUserId, + mship: "join", user: otherUserId, }); const redaction = utils.mkEvent({ type: "m.room.redaction", - room_id: roomId, sender: otherUserId, content: {}, }); @@ -987,7 +1030,7 @@ describe("MatrixClient event timelines", function() { timeline: { events: [ utils.mkMessage({ - room: roomId, user: otherUserId, msg: "world", + user: otherUserId, msg: "world", }), ], limited: true, @@ -1006,4 +1049,75 @@ describe("MatrixClient event timelines", function() { expect(tl.getEvents().length).toEqual(1); }); }); + + it("should re-insert room IDs for bundled thread relation events", async () => { + // @ts-ignore + client.clientOpts.experimentalThreadSupport = true; + Thread.setServerSideSupport(true, false); + + httpBackend.when("GET", "/sync").respond(200, { + next_batch: "s_5_4", + rooms: { + join: { + [roomId]: { + timeline: { + events: [ + SYNC_THREAD_ROOT, + ], + prev_batch: "f_1_1", + }, + }, + }, + }, + }); + await Promise.all([httpBackend.flushAllExpected(), utils.syncPromise(client)]); + + const room = client.getRoom(roomId); + const thread = room.getThread(THREAD_ROOT.event_id); + const timelineSet = thread.timelineSet; + + httpBackend.when("GET", "/rooms/!foo%3Abar/context/" + encodeURIComponent(THREAD_ROOT.event_id)) + .respond(200, { + start: "start_token", + events_before: [], + event: THREAD_ROOT, + events_after: [], + state: [], + end: "end_token", + }); + httpBackend.when("GET", "/rooms/!foo%3Abar/relations/" + + encodeURIComponent(THREAD_ROOT.event_id) + "/" + + encodeURIComponent(THREAD_RELATION_TYPE.name) + "?limit=20") + .respond(200, function() { + return { + original_event: THREAD_ROOT, + chunk: [THREAD_REPLY], + // no next batch as this is the oldest end of the timeline + }; + }); + await Promise.all([ + client.getEventTimeline(timelineSet, THREAD_ROOT.event_id), + httpBackend.flushAllExpected(), + ]); + + httpBackend.when("GET", "/sync").respond(200, { + next_batch: "s_5_5", + rooms: { + join: { + [roomId]: { + timeline: { + events: [ + SYNC_THREAD_REPLY, + ], + prev_batch: "f_1_2", + }, + }, + }, + }, + }); + + await Promise.all([httpBackend.flushAllExpected(), utils.syncPromise(client)]); + + expect(thread.liveTimeline.getEvents()[1].event).toEqual(THREAD_REPLY); + }); }); diff --git a/spec/integ/matrix-client-methods.spec.js b/spec/integ/matrix-client-methods.spec.js index 5c9855b2ea6..0dd33a02c3d 100644 --- a/spec/integ/matrix-client-methods.spec.js +++ b/spec/integ/matrix-client-methods.spec.js @@ -27,11 +27,19 @@ describe("MatrixClient", function() { let store = null; const userId = "@alice:localhost"; const accessToken = "aseukfgwef"; + const idServerDomain = "identity.localhost"; // not a real server + const identityAccessToken = "woop-i-am-a-secret"; beforeEach(function() { store = new MemoryStore(); - const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, { store }); + const testClient = new TestClient(userId, "aliceDevice", accessToken, undefined, { + store, + identityServer: { + getAccessToken: () => Promise.resolve(identityAccessToken), + }, + idBaseUrl: `https://${idServerDomain}`, + }); httpBackend = testClient.httpBackend; client = testClient.client; }); @@ -993,7 +1001,7 @@ describe("MatrixClient", function() { }; httpBackend.when("GET", "/_matrix/client/versions").respond(200, { - versions: ["r0.5.0"], + versions: ["r0.6.0"], }); const prom = client.requestRegisterEmailToken("bob@email", "secret", 1); @@ -1008,6 +1016,64 @@ describe("MatrixClient", function() { expect(await prom).toStrictEqual(response); }); }); + + describe("inviteByThreePid", () => { + it("should supply an id_access_token", async () => { + const targetEmail = "gerald@example.org"; + + httpBackend.when("GET", "/_matrix/client/versions").respond(200, { + versions: ["r0.6.0"], + }); + + httpBackend.when("POST", "/invite").check(req => { + expect(req.data).toStrictEqual({ + id_server: idServerDomain, + id_access_token: identityAccessToken, + medium: "email", + address: targetEmail, + }); + }).respond(200, {}); + + const prom = client.inviteByThreePid("!room:example.org", "email", targetEmail); + await httpBackend.flush(); + await prom; // returns empty object, so no validation needed + }); + }); + + describe("createRoom", () => { + it("should populate id_access_token on 3pid invites", async () => { + const targetEmail = "gerald@example.org"; + const response = { + room_id: "!room:localhost", + }; + const input = { + invite_3pid: [{ + // we intentionally exclude the access token here, so it can be populated for us + id_server: idServerDomain, + medium: "email", + address: targetEmail, + }], + }; + + httpBackend.when("GET", "/_matrix/client/versions").respond(200, { + versions: ["r0.6.0"], + }); + + httpBackend.when("POST", "/createRoom").check(req => { + expect(req.data).toMatchObject({ + invite_3pid: expect.arrayContaining([{ + ...input.invite_3pid[0], + id_access_token: identityAccessToken, + }]), + }); + expect(req.data.invite_3pid.length).toBe(1); + }).respond(200, response); + + const prom = client.createRoom(input); + await httpBackend.flush(); + expect(await prom).toStrictEqual(response); + }); + }); }); function withThreadId(event, newThreadId) { diff --git a/spec/test-utils/test-utils.ts b/spec/test-utils/test-utils.ts index 84a9662e419..4e0a311a0aa 100644 --- a/spec/test-utils/test-utils.ts +++ b/spec/test-utils/test-utils.ts @@ -70,7 +70,7 @@ export function mock(constr: { new(...args: any[]): T }, name: string): T { interface IEventOpts { type: EventType | string; - room: string; + room?: string; sender?: string; skey?: string; content: IContent; @@ -93,8 +93,8 @@ let testEventIndex = 1; // counter for events, easier for comparison of randomly * @return {Object} a JSON object representing this event. */ export function mkEvent(opts: IEventOpts & { event: true }, client?: MatrixClient): MatrixEvent; -export function mkEvent(opts: IEventOpts & { event?: false }, client?: MatrixClient): object; -export function mkEvent(opts: IEventOpts & { event?: boolean }, client?: MatrixClient): object | MatrixEvent { +export function mkEvent(opts: IEventOpts & { event?: false }, client?: MatrixClient): Partial; +export function mkEvent(opts: IEventOpts & { event?: boolean }, client?: MatrixClient): Partial | MatrixEvent { if (!opts.type || !opts.content) { throw new Error("Missing .type or .content =>" + JSON.stringify(opts)); } @@ -145,8 +145,8 @@ interface IPresenceOpts { * @return {Object|MatrixEvent} The event */ export function mkPresence(opts: IPresenceOpts & { event: true }): MatrixEvent; -export function mkPresence(opts: IPresenceOpts & { event?: false }): object; -export function mkPresence(opts: IPresenceOpts & { event?: boolean }): object | MatrixEvent { +export function mkPresence(opts: IPresenceOpts & { event?: false }): Partial; +export function mkPresence(opts: IPresenceOpts & { event?: boolean }): Partial | MatrixEvent { const event = { event_id: "$" + Math.random() + "-" + Math.random(), type: "m.presence", @@ -162,7 +162,7 @@ export function mkPresence(opts: IPresenceOpts & { event?: boolean }): object | } interface IMembershipOpts { - room: string; + room?: string; mship: string; sender?: string; user?: string; @@ -186,8 +186,8 @@ interface IMembershipOpts { * @return {Object|MatrixEvent} The event */ export function mkMembership(opts: IMembershipOpts & { event: true }): MatrixEvent; -export function mkMembership(opts: IMembershipOpts & { event?: false }): object; -export function mkMembership(opts: IMembershipOpts & { event?: boolean }): object | MatrixEvent { +export function mkMembership(opts: IMembershipOpts & { event?: false }): Partial; +export function mkMembership(opts: IMembershipOpts & { event?: boolean }): Partial | MatrixEvent { const eventOpts: IEventOpts = { ...opts, type: EventType.RoomMember, @@ -209,7 +209,7 @@ export function mkMembership(opts: IMembershipOpts & { event?: boolean }): objec } interface IMessageOpts { - room: string; + room?: string; user: string; msg?: string; event?: boolean; @@ -226,8 +226,11 @@ interface IMessageOpts { * @return {Object|MatrixEvent} The event */ export function mkMessage(opts: IMessageOpts & { event: true }, client?: MatrixClient): MatrixEvent; -export function mkMessage(opts: IMessageOpts & { event?: false }, client?: MatrixClient): object; -export function mkMessage(opts: IMessageOpts & { event?: boolean }, client?: MatrixClient): object | MatrixEvent { +export function mkMessage(opts: IMessageOpts & { event?: false }, client?: MatrixClient): Partial; +export function mkMessage( + opts: IMessageOpts & { event?: boolean }, + client?: MatrixClient, +): Partial | MatrixEvent { const eventOpts: IEventOpts = { ...opts, type: EventType.RoomMessage, @@ -260,11 +263,11 @@ interface IReplyMessageOpts extends IMessageOpts { * @return {Object|MatrixEvent} The event */ export function mkReplyMessage(opts: IReplyMessageOpts & { event: true }, client?: MatrixClient): MatrixEvent; -export function mkReplyMessage(opts: IReplyMessageOpts & { event?: false }, client?: MatrixClient): object; +export function mkReplyMessage(opts: IReplyMessageOpts & { event?: false }, client?: MatrixClient): Partial; export function mkReplyMessage( opts: IReplyMessageOpts & { event?: boolean }, client?: MatrixClient, -): object | MatrixEvent { +): Partial | MatrixEvent { const eventOpts: IEventOpts = { ...opts, type: EventType.RoomMessage, diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts new file mode 100644 index 00000000000..8b59469126b --- /dev/null +++ b/spec/test-utils/webrtc.ts @@ -0,0 +1,157 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export const DUMMY_SDP = ( + "v=0\r\n" + + "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" + + "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" + + "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" + + "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" + + "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" + + "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" + + "a=ice-options:trickle\r\n" + + "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" + + "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" + + "a=setup:active\r\n" + + "a=mid:0\r\n" + + "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" + + "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" + + "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" + + "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" + + "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" + + "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" + + "a=sendrecv\r\n" + + "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" + + "a=rtcp-mux\r\n" + + "a=rtpmap:111 opus/48000/2\r\n" + + "a=rtcp-fb:111 transport-cc\r\n" + + "a=fmtp:111 minptime=10;useinbandfec=1\r\n" + + "a=rtpmap:103 ISAC/16000\r\n" + + "a=rtpmap:104 ISAC/32000\r\n" + + "a=rtpmap:9 G722/8000\r\n" + + "a=rtpmap:0 PCMU/8000\r\n" + + "a=rtpmap:8 PCMA/8000\r\n" + + "a=rtpmap:106 CN/32000\r\n" + + "a=rtpmap:105 CN/16000\r\n" + + "a=rtpmap:13 CN/8000\r\n" + + "a=rtpmap:110 telephone-event/48000\r\n" + + "a=rtpmap:112 telephone-event/32000\r\n" + + "a=rtpmap:113 telephone-event/16000\r\n" + + "a=rtpmap:126 telephone-event/8000\r\n" + + "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" +); + +class MockMediaStreamAudioSourceNode { + connect() {} +} + +export class MockAudioContext { + constructor() {} + createAnalyser() { return {}; } + createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } + close() {} +} + +export class MockRTCPeerConnection { + localDescription: RTCSessionDescription; + + constructor() { + this.localDescription = { + sdp: DUMMY_SDP, + type: 'offer', + toJSON: function() { }, + }; + } + + addEventListener() { } + createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } + createOffer() { + return Promise.resolve({}); + } + setRemoteDescription() { + return Promise.resolve(); + } + setLocalDescription() { + return Promise.resolve(); + } + close() { } + getStats() { return []; } + addTrack(track: MockMediaStreamTrack) { return new MockRTCRtpSender(track); } +} + +export class MockRTCRtpSender { + constructor(public track: MockMediaStreamTrack) { } + + replaceTrack(track: MockMediaStreamTrack) { this.track = track; } +} + +export class MockMediaStreamTrack { + constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { } + + stop() { } +} + +// XXX: Using EventTarget in jest doesn't seem to work, so we write our own +// implementation +export class MockMediaStream { + constructor( + public id: string, + private tracks: MockMediaStreamTrack[] = [], + ) {} + + listeners: [string, (...args: any[]) => any][] = []; + + dispatchEvent(eventType: string) { + this.listeners.forEach(([t, c]) => { + if (t !== eventType) return; + c(); + }); + } + getTracks() { return this.tracks; } + getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); } + getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); } + addEventListener(eventType: string, callback: (...args: any[]) => any) { + this.listeners.push([eventType, callback]); + } + removeEventListener(eventType: string, callback: (...args: any[]) => any) { + this.listeners.filter(([t, c]) => { + return t !== eventType || c !== callback; + }); + } + addTrack(track: MockMediaStreamTrack) { + this.tracks.push(track); + this.dispatchEvent("addtrack"); + } + removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } +} + +export class MockMediaDeviceInfo { + constructor( + public kind: "audio" | "video", + ) { } +} + +export class MockMediaHandler { + getUserMediaStream(audio: boolean, video: boolean) { + const tracks = []; + if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); + if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); + + return new MockMediaStream("mock_stream_from_media_handler", tracks); + } + stopUserMediaStream() { } + hasAudioDevice() { return true; } +} diff --git a/spec/unit/crypto/backup.spec.js b/spec/unit/crypto/backup.spec.ts similarity index 78% rename from spec/unit/crypto/backup.spec.js rename to spec/unit/crypto/backup.spec.ts index cab0c0d0d9f..6759fe16152 100644 --- a/spec/unit/crypto/backup.spec.js +++ b/spec/unit/crypto/backup.spec.ts @@ -15,6 +15,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { MockedObject } from "jest-mock"; + import '../../olm-loader'; import { logger } from "../../../src/logger"; import * as olmlib from "../../../src/crypto/olmlib"; @@ -22,12 +24,13 @@ import { MatrixClient } from "../../../src/client"; import { MatrixEvent } from "../../../src/models/event"; import * as algorithms from "../../../src/crypto/algorithms"; import { MemoryCryptoStore } from "../../../src/crypto/store/memory-crypto-store"; -import { MockStorageApi } from "../../MockStorageApi"; import * as testUtils from "../../test-utils/test-utils"; import { OlmDevice } from "../../../src/crypto/OlmDevice"; import { Crypto } from "../../../src/crypto"; import { resetCrossSigningKeys } from "./crypto-utils"; import { BackupManager } from "../../../src/crypto/backup"; +import { StubStore } from "../../../src/store/stub"; +import { IAbortablePromise, MatrixScheduler } from '../../../src'; const Olm = global.Olm; @@ -92,8 +95,8 @@ const AES256_KEY_BACKUP_DATA = { }; const CURVE25519_BACKUP_INFO = { - algorithm: "m.megolm_backup.v1.curve25519-aes-sha2", - version: 1, + algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM, + version: '1', auth_data: { public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo", }, @@ -101,7 +104,7 @@ const CURVE25519_BACKUP_INFO = { const AES256_BACKUP_INFO = { algorithm: "org.matrix.msc3270.v1.aes-hmac-sha2", - version: 1, + version: '1', auth_data: { // FIXME: add iv and mac }, @@ -121,21 +124,14 @@ function makeTestClient(cryptoStore) { const scheduler = [ "getQueueForEvent", "queueEvent", "removeEventFromQueue", "setProcessFunction", - ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}); - const store = [ - "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback", - "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", - "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter", - "storeFilter", "getSyncAccumulator", "startup", "deleteAllData", - ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}); - store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null)); - store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null)); - store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null)); + ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}) as MockedObject; + const store = new StubStore(); + return new MatrixClient({ baseUrl: "https://my.home.server", idBaseUrl: "https://identity.server", accessToken: "my.access.token", - request: function() {}, // NOP + request: jest.fn(), // NOP store: store, scheduler: scheduler, userId: "@alice:bar", @@ -158,7 +154,6 @@ describe("MegolmBackup", function() { let olmDevice; let mockOlmLib; let mockCrypto; - let mockStorage; let cryptoStore; let megolmDecryption; beforeEach(async function() { @@ -170,8 +165,7 @@ describe("MegolmBackup", function() { ); mockCrypto.backupInfo = CURVE25519_BACKUP_INFO; - mockStorage = new MockStorageApi(); - cryptoStore = new MemoryCryptoStore(mockStorage); + cryptoStore = new MemoryCryptoStore(); olmDevice = new OlmDevice(cryptoStore); @@ -184,7 +178,6 @@ describe("MegolmBackup", function() { describe("backup", function() { let mockBaseApis; - let realSetTimeout; beforeEach(function() { mockBaseApis = {}; @@ -202,14 +195,14 @@ describe("MegolmBackup", function() { // clobber the setTimeout function to run 100x faster. // ideally we would use lolex, but we have no oportunity // to tick the clock between the first try and the retry. - realSetTimeout = global.setTimeout; - global.setTimeout = function(f, n) { + const realSetTimeout = global.setTimeout; + jest.spyOn(global, 'setTimeout').mockImplementation(function(f, n) { return realSetTimeout(f, n/100); - }; + }); }); afterEach(function() { - global.setTimeout = realSetTimeout; + jest.spyOn(global, 'setTimeout').mockRestore(); }); it('automatically calls the key back up', function() { @@ -289,16 +282,16 @@ describe("MegolmBackup", function() { txn); }); }) - .then(() => { - client.enableKeyBackup({ - algorithm: "m.megolm_backup.v1.curve25519-aes-sha2", - version: 1, + .then(async () => { + await client.enableKeyBackup({ + algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM, + version: '1', auth_data: { public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo", }, }); let numCalls = 0; - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { client.http.authedRequest = function( callback, method, path, queryParams, data, opts, ) { @@ -307,17 +300,17 @@ describe("MegolmBackup", function() { if (numCalls >= 2) { // exit out of retry loop if there's something wrong reject(new Error("authedRequest called too many timmes")); - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; } expect(method).toBe("PUT"); expect(path).toBe("/room_keys/keys"); - expect(queryParams.version).toBe(1); + expect(queryParams.version).toBe('1'); expect(data.rooms[ROOM_ID].sessions).toBeDefined(); expect(data.rooms[ROOM_ID].sessions).toHaveProperty( groupSession.session_id(), ); resolve(); - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; }; client.crypto.backupManager.backupGroupSession( "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI", @@ -371,17 +364,17 @@ describe("MegolmBackup", function() { txn); }); }) - .then(() => { - client.enableKeyBackup({ + .then(async () => { + await client.enableKeyBackup({ algorithm: "org.matrix.msc3270.v1.aes-hmac-sha2", - version: 1, + version: '1', auth_data: { iv: "PsCAtR7gMc4xBd9YS3A9Ow", mac: "ZSDsTFEZK7QzlauCLMleUcX96GQZZM7UNtk4sripSqQ", }, }); let numCalls = 0; - return new Promise((resolve, reject) => { + return new Promise((resolve, reject) => { client.http.authedRequest = function( callback, method, path, queryParams, data, opts, ) { @@ -390,17 +383,17 @@ describe("MegolmBackup", function() { if (numCalls >= 2) { // exit out of retry loop if there's something wrong reject(new Error("authedRequest called too many timmes")); - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; } expect(method).toBe("PUT"); expect(path).toBe("/room_keys/keys"); - expect(queryParams.version).toBe(1); + expect(queryParams.version).toBe('1'); expect(data.rooms[ROOM_ID].sessions).toBeDefined(); expect(data.rooms[ROOM_ID].sessions).toHaveProperty( groupSession.session_id(), ); resolve(); - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; }; client.crypto.backupManager.backupGroupSession( "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI", @@ -432,19 +425,12 @@ describe("MegolmBackup", function() { megolmDecryption.olmlib = mockOlmLib; await client.initCrypto(); - let privateKeys; - client.uploadDeviceSigningKeys = async function(e) {return;}; - client.uploadKeySignatures = async function(e) {return;}; - client.on("crossSigning.saveCrossSigningKeys", function(e) { - privateKeys = e; - }); - client.on("crossSigning.getKey", function(e) { - e.done(privateKeys[e.type]); - }); + client.uploadDeviceSigningKeys = async function(e) {return {};}; + client.uploadKeySignatures = async function(e) {return { failures: {} };}; await resetCrossSigningKeys(client); let numCalls = 0; await Promise.all([ - new Promise((resolve, reject) => { + new Promise((resolve, reject) => { let backupInfo; client.http.authedRequest = function( callback, method, path, queryParams, data, opts, @@ -461,24 +447,24 @@ describe("MegolmBackup", function() { ); } catch (e) { reject(e); - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; } backupInfo = data; - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; } else if (numCalls === 2) { expect(method).toBe("GET"); expect(path).toBe("/room_keys/version"); resolve(); - return Promise.resolve(backupInfo); + return Promise.resolve(backupInfo) as IAbortablePromise; } else { // exit out of retry loop if there's something wrong reject(new Error("authedRequest called too many times")); - return Promise.resolve({}); + return Promise.resolve({}) as IAbortablePromise; } }; }), client.createKeyBackupVersion({ - algorithm: "m.megolm_backup.v1.curve25519-aes-sha2", + algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM, auth_data: { public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo", }, @@ -488,7 +474,7 @@ describe("MegolmBackup", function() { client.stopClient(); }); - it('retries when a backup fails', function() { + it('retries when a backup fails', async function() { const groupSession = new Olm.OutboundGroupSession(); groupSession.create(); const ibGroupSession = new Olm.InboundGroupSession(); @@ -497,21 +483,13 @@ describe("MegolmBackup", function() { const scheduler = [ "getQueueForEvent", "queueEvent", "removeEventFromQueue", "setProcessFunction", - ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}); - const store = [ - "getRoom", "getRooms", "getUser", "getSyncToken", "scrollback", - "save", "wantsSave", "setSyncToken", "storeEvents", "storeRoom", - "storeUser", "getFilterIdByName", "setFilterIdByName", "getFilter", - "storeFilter", "getSyncAccumulator", "startup", "deleteAllData", - ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}); - store.getSavedSync = jest.fn().mockReturnValue(Promise.resolve(null)); - store.getSavedSyncToken = jest.fn().mockReturnValue(Promise.resolve(null)); - store.setSyncData = jest.fn().mockReturnValue(Promise.resolve(null)); + ].reduce((r, k) => {r[k] = jest.fn(); return r;}, {}) as MockedObject; + const store = new StubStore(); const client = new MatrixClient({ baseUrl: "https://my.home.server", idBaseUrl: "https://identity.server", accessToken: "my.access.token", - request: function() {}, // NOP + request: jest.fn(), // NOP store: store, scheduler: scheduler, userId: "@alice:bar", @@ -529,71 +507,68 @@ describe("MegolmBackup", function() { megolmDecryption.olmlib = mockOlmLib; - return client.initCrypto() - .then(() => { - return cryptoStore.doTxn( - "readwrite", - [cryptoStore.STORE_SESSION], - (txn) => { - cryptoStore.addEndToEndInboundGroupSession( - "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI", - groupSession.session_id(), - { - forwardingCurve25519KeyChain: undefined, - keysClaimed: { - ed25519: "SENDER_ED25519", - }, - room_id: ROOM_ID, - session: ibGroupSession.pickle(olmDevice.pickleKey), - }, - txn); - }); - }) - .then(() => { - client.enableKeyBackup({ - algorithm: "m.megolm_backup.v1.curve25519-aes-sha2", - version: 1, - auth_data: { - public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo", + await client.initCrypto(); + await cryptoStore.doTxn( + "readwrite", + [cryptoStore.STORE_SESSION], + (txn) => { + cryptoStore.addEndToEndInboundGroupSession( + "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI", + groupSession.session_id(), + { + forwardingCurve25519KeyChain: undefined, + keysClaimed: { + ed25519: "SENDER_ED25519", + }, + room_id: ROOM_ID, + session: ibGroupSession.pickle(olmDevice.pickleKey), }, - }); - let numCalls = 0; - return new Promise((resolve, reject) => { - client.http.authedRequest = function( - callback, method, path, queryParams, data, opts, - ) { - ++numCalls; - expect(numCalls).toBeLessThanOrEqual(2); - if (numCalls >= 3) { - // exit out of retry loop if there's something wrong - reject(new Error("authedRequest called too many timmes")); - return Promise.resolve({}); - } - expect(method).toBe("PUT"); - expect(path).toBe("/room_keys/keys"); - expect(queryParams.version).toBe(1); - expect(data.rooms[ROOM_ID].sessions).toBeDefined(); - expect(data.rooms[ROOM_ID].sessions).toHaveProperty( - groupSession.session_id(), - ); - if (numCalls > 1) { - resolve(); - return Promise.resolve({}); - } else { - return Promise.reject( - new Error("this is an expected failure"), - ); - } - }; - client.crypto.backupManager.backupGroupSession( - "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI", - groupSession.session_id(), - ); - }).then(() => { - expect(numCalls).toBe(2); - client.stopClient(); - }); + txn); }); + + await client.enableKeyBackup({ + algorithm: olmlib.MEGOLM_BACKUP_ALGORITHM, + version: '1', + auth_data: { + public_key: "hSDwCYkwp1R0i33ctD73Wg2/Og0mOBr066SpjqqbTmo", + }, + }); + let numCalls = 0; + + await new Promise((resolve, reject) => { + client.http.authedRequest = function( + callback, method, path, queryParams, data, opts, + ) { + ++numCalls; + expect(numCalls).toBeLessThanOrEqual(2); + if (numCalls >= 3) { + // exit out of retry loop if there's something wrong + reject(new Error("authedRequest called too many timmes")); + return Promise.resolve({}) as IAbortablePromise; + } + expect(method).toBe("PUT"); + expect(path).toBe("/room_keys/keys"); + expect(queryParams.version).toBe('1'); + expect(data.rooms[ROOM_ID].sessions).toBeDefined(); + expect(data.rooms[ROOM_ID].sessions).toHaveProperty( + groupSession.session_id(), + ); + if (numCalls > 1) { + resolve(); + return Promise.resolve({}) as IAbortablePromise; + } else { + return Promise.reject( + new Error("this is an expected failure"), + ) as IAbortablePromise; + } + }; + return client.crypto.backupManager.backupGroupSession( + "F0Q2NmyJNgUVj9DGsb4ZQt3aVxhVcUQhg7+gvW0oyKI", + groupSession.session_id(), + ); + }); + expect(numCalls).toBe(2); + client.stopClient(); }); }); diff --git a/spec/unit/crypto/crypto-utils.js b/spec/unit/crypto/crypto-utils.ts similarity index 78% rename from spec/unit/crypto/crypto-utils.js rename to spec/unit/crypto/crypto-utils.ts index ecc6fc4b0ae..3535edaabe7 100644 --- a/spec/unit/crypto/crypto-utils.js +++ b/spec/unit/crypto/crypto-utils.ts @@ -1,11 +1,13 @@ +import { IRecoveryKey } from '../../../src/crypto/api'; +import { CrossSigningLevel } from '../../../src/crypto/CrossSigning'; import { IndexedDBCryptoStore } from '../../../src/crypto/store/indexeddb-crypto-store'; // needs to be phased out and replaced with bootstrapSecretStorage, // but that is doing too much extra stuff for it to be an easy transition. -export async function resetCrossSigningKeys(client, { - level, - authUploadDeviceSigningKeys = async func => await func(), -} = {}) { +export async function resetCrossSigningKeys( + client, + { level }: { level?: CrossSigningLevel} = {}, +): Promise { const crypto = client.crypto; const oldKeys = Object.assign({}, crypto.crossSigningInfo.keys); @@ -30,14 +32,14 @@ export async function resetCrossSigningKeys(client, { await crypto.afterCrossSigningLocalKeyChange(); } -export async function createSecretStorageKey() { +export async function createSecretStorageKey(): Promise { const decryption = new global.Olm.PkDecryption(); const storagePublicKey = decryption.generate_key(); const storagePrivateKey = decryption.get_private_key(); decryption.free(); return { // `pubkey` not used anymore with symmetric 4S - keyInfo: { pubkey: storagePublicKey }, + keyInfo: { pubkey: storagePublicKey, key: undefined }, privateKey: storagePrivateKey, }; } diff --git a/spec/unit/realtime-callbacks.spec.js b/spec/unit/realtime-callbacks.spec.js index f99152968b2..8e57e48e6f1 100644 --- a/spec/unit/realtime-callbacks.spec.js +++ b/spec/unit/realtime-callbacks.spec.js @@ -1,7 +1,7 @@ import * as callbacks from "../../src/realtime-callbacks"; let wallTime = 1234567890; -jest.useFakeTimers(); +jest.useFakeTimers().setSystemTime(wallTime); describe("realtime-callbacks", function() { function tick(millis) { @@ -9,14 +9,6 @@ describe("realtime-callbacks", function() { jest.advanceTimersByTime(millis); } - beforeEach(function() { - callbacks.setNow(() => wallTime); - }); - - afterEach(function() { - callbacks.setNow(); - }); - describe("setTimeout", function() { it("should call the callback after the timeout", function() { const callback = jest.fn(); diff --git a/spec/unit/utils.spec.ts b/spec/unit/utils.spec.ts index 340acf92d2f..03f663ab39c 100644 --- a/spec/unit/utils.spec.ts +++ b/spec/unit/utils.spec.ts @@ -109,18 +109,6 @@ describe("utils", function() { }); }); - describe("checkObjectHasNoAdditionalKeys", function() { - it("should throw for extra keys", function() { - expect(function() { - utils.checkObjectHasNoAdditionalKeys({ foo: "bar", baz: 4 }, ["foo"]); - }).toThrow(); - - expect(function() { - utils.checkObjectHasNoAdditionalKeys({ foo: "bar" }, ["foo"]); - }).not.toThrow(); - }); - }); - describe("deepCompare", function() { const assert = { isTrue: function(x: any) { diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 9195327be0f..6fc8a5a5d81 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -1,5 +1,5 @@ /* -Copyright 2020 The Matrix.org Foundation C.I.C. +Copyright 2020 - 2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -15,130 +15,26 @@ limitations under the License. */ import { TestClient } from '../../TestClient'; -import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall } from '../../../src/webrtc/call'; +import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call'; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; -import { RoomMember } from "../../../src"; - -const DUMMY_SDP = ( - "v=0\r\n" + - "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" + - "s=-\r\nt=0 0\r\na=group:BUNDLE 0\r\n" + - "a=msid-semantic: WMS h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA\r\n" + - "m=audio 9 UDP/TLS/RTP/SAVPF 111 103 104 9 0 8 106 105 13 110 112 113 126\r\n" + - "c=IN IP4 0.0.0.0\r\na=rtcp:9 IN IP4 0.0.0.0\r\na=ice-ufrag:hLDR\r\n" + - "a=ice-pwd:bMGD9aOldHWiI+6nAq/IIlRw\r\n" + - "a=ice-options:trickle\r\n" + - "a=fingerprint:sha-256 E4:94:84:F9:4A:98:8A:56:F5:5F:FD:AF:72:B9:32:89:49:5C:4B:9A:" + - "4A:15:8E:41:8A:F3:69:E4:39:52:DC:D6\r\n" + - "a=setup:active\r\n" + - "a=mid:0\r\n" + - "a=extmap:1 urn:ietf:params:rtp-hdrext:ssrc-audio-level\r\n" + - "a=extmap:2 http://www.webrtc.org/experiments/rtp-hdrext/abs-send-time\r\n" + - "a=extmap:3 http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01\r\n" + - "a=extmap:4 urn:ietf:params:rtp-hdrext:sdes:mid\r\n" + - "a=extmap:5 urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id\r\n" + - "a=extmap:6 urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id\r\n" + - "a=sendrecv\r\n" + - "a=msid:h3wAi7s8QpiQMH14WG3BnDbmlOqo9I5ezGZA 4357098f-3795-4131-bff4-9ba9c0348c49\r\n" + - "a=rtcp-mux\r\n" + - "a=rtpmap:111 opus/48000/2\r\n" + - "a=rtcp-fb:111 transport-cc\r\n" + - "a=fmtp:111 minptime=10;useinbandfec=1\r\n" + - "a=rtpmap:103 ISAC/16000\r\n" + - "a=rtpmap:104 ISAC/32000\r\n" + - "a=rtpmap:9 G722/8000\r\n" + - "a=rtpmap:0 PCMU/8000\r\n" + - "a=rtpmap:8 PCMA/8000\r\n" + - "a=rtpmap:106 CN/32000\r\n" + - "a=rtpmap:105 CN/16000\r\n" + - "a=rtpmap:13 CN/8000\r\n" + - "a=rtpmap:110 telephone-event/48000\r\n" + - "a=rtpmap:112 telephone-event/32000\r\n" + - "a=rtpmap:113 telephone-event/16000\r\n" + - "a=rtpmap:126 telephone-event/8000\r\n" + - "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" -); - -class MockMediaStreamAudioSourceNode { - connect() {} -} - -class MockAudioContext { - constructor() {} - createAnalyser() { return {}; } - createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } - close() {} -} - -class MockRTCPeerConnection { - localDescription: RTCSessionDescription; - - constructor() { - this.localDescription = { - sdp: DUMMY_SDP, - type: 'offer', - toJSON: function() {}, - }; - } - - addEventListener() {} - createOffer() { - return Promise.resolve({}); - } - setRemoteDescription() { - return Promise.resolve(); - } - setLocalDescription() { - return Promise.resolve(); - } - close() {} - getStats() { return []; } - addTrack(track: MockMediaStreamTrack) {return new MockRTCRtpSender(track);} -} - -class MockRTCRtpSender { - constructor(public track: MockMediaStreamTrack) {} - - replaceTrack(track: MockMediaStreamTrack) {this.track = track;} -} - -class MockMediaStreamTrack { - constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) {} - - stop() {} -} - -class MockMediaStream { - constructor( - public id: string, - private tracks: MockMediaStreamTrack[] = [], - ) {} - - getTracks() { return this.tracks; } - getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); } - getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); } - addEventListener() {} - removeEventListener() { } - addTrack(track: MockMediaStreamTrack) {this.tracks.push(track);} - removeTrack(track: MockMediaStreamTrack) {this.tracks.splice(this.tracks.indexOf(track), 1);} -} - -class MockMediaDeviceInfo { - constructor( - public kind: "audio" | "video", - ) {} -} - -class MockMediaHandler { - getUserMediaStream(audio: boolean, video: boolean) { - const tracks = []; - if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); - if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); - - return new MockMediaStream("mock_stream_from_media_handler", tracks); - } - stopUserMediaStream() {} -} +import { + DUMMY_SDP, + MockMediaHandler, + MockMediaStream, + MockMediaStreamTrack, + MockMediaDeviceInfo, + MockRTCPeerConnection, + MockAudioContext, +} from "../../test-utils/webrtc"; +import { CallFeed } from "../../../src/webrtc/callFeed"; + +const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { + const callPromise = call.placeVoiceCall(); + await client.httpBackend.flush(""); + await callPromise; + + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); +}; describe('Call', function() { let client; @@ -207,9 +103,8 @@ describe('Call', function() { }); it('should ignore candidate events from non-matching party ID', async function() { - const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); - await callPromise; + await startVoiceCall(client, call); + await call.onAnswerReceived({ getContent: () => { return { @@ -266,9 +161,7 @@ describe('Call', function() { }); it('should add candidates received before answer if party ID is correct', async function() { - const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); - await callPromise; + await startVoiceCall(client, call); call.peerConn.addIceCandidate = jest.fn(); call.onRemoteIceCandidatesReceived({ @@ -329,9 +222,7 @@ describe('Call', function() { }); it('should map asserted identity messages to remoteAssertedIdentity', async function() { - const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); - await callPromise; + await startVoiceCall(client, call); await call.onAnswerReceived({ getContent: () => { return { @@ -375,13 +266,7 @@ describe('Call', function() { }); it("should map SDPStreamMetadata to feeds", async () => { - const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); - await callPromise; - - call.getOpponentMember = () => { - return { userId: "@bob:bar.uk" }; - }; + await startVoiceCall(client, call); await call.onAnswerReceived({ getContent: () => { @@ -420,13 +305,7 @@ describe('Call', function() { }); it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { - const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); - await callPromise; - - call.getOpponentMember = () => { - return { userId: "@bob:bar.uk" } as RoomMember; - }; + await startVoiceCall(client, call); await call.onAnswerReceived({ getContent: () => { @@ -470,9 +349,7 @@ describe('Call', function() { ), ); - const callPromise = call.placeVideoCall(); - await client.httpBackend.flush(); - await callPromise; + await startVoiceCall(client, call); await call.onAnswerReceived({ getContent: () => { @@ -509,9 +386,7 @@ describe('Call', function() { }); it("should handle upgrade to video call", async () => { - const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); - await callPromise; + await startVoiceCall(client, call); await call.onAnswerReceived({ getContent: () => { @@ -540,6 +415,350 @@ describe('Call', function() { }).track.id).toBe("video_track"); }); + describe("should handle stream replacement", () => { + it("with both purpose and id", async () => { + await startVoiceCall(client, call); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream1": { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1"); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream2": { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream2", [])); + + expect(feed?.stream?.id).toBe("remote_stream2"); + }); + + it("with just purpose", async () => { + await startVoiceCall(client, call); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream1": { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1"); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream2": { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream2", [])); + + expect(feed?.stream?.id).toBe("remote_stream2"); + }); + + it("should not replace purpose is different", async () => { + await startVoiceCall(client, call); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream1": { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1"); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream2": { + purpose: SDPStreamMetadataPurpose.Screenshare, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream2", [])); + + expect(feed?.stream?.id).toBe("remote_stream1"); + }); + }); + + it("should handle SDPStreamMetadata changes", async () => { + await startVoiceCall(client, call); + + call.updateRemoteSDPStreamMetadata({ + "remote_stream": { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: false, + video_muted: false, + }, + }); + call.pushRemoteFeed(new MockMediaStream("remote_stream", [])); + const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); + + call.onSDPStreamMetadataChangedReceived({ + getContent: () => ({ + [SDPStreamMetadataKey]: { + "remote_stream": { + purpose: SDPStreamMetadataPurpose.Screenshare, + audio_muted: true, + video_muted: true, + id: "feed_id2", + }, + }, + }), + }); + + expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Screenshare); + expect(feed?.audioMuted).toBe(true); + expect(feed?.videoMuted).toBe(true); + }); + + it("should choose opponent member", async () => { + const callPromise = call.placeVoiceCall(); + await client.httpBackend.flush(); + await callPromise; + + const opponentMember = { + roomId: call.roomId, + userId: "opponentUserId", + }; + + client.client.getRoom = () => { + return { + getMember: (userId) => { + if (userId === opponentMember.userId) { + return opponentMember; + } + }, + }; + }; + + const opponentCaps = { + "m.call.transferee": true, + "m.call.dtmf": false, + }; + call.chooseOpponent({ + getContent: () => ({ + version: 1, + party_id: "party_id", + capabilities: opponentCaps, + }), + getSender: () => opponentMember.userId, + }); + + expect(call.getOpponentMember()).toBe(opponentMember); + expect(call.opponentPartyId).toBe("party_id"); + expect(call.opponentCaps).toBe(opponentCaps); + expect(call.opponentCanBeTransferred()).toBe(true); + expect(call.opponentSupportsDTMF()).toBe(false); + }); + + describe("should deduce the call type correctly", () => { + it("if no video", async () => { + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + + call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + expect(call.type).toBe(CallType.Voice); + }); + + it("if remote video", async () => { + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + + call.pushRemoteFeed(new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")])); + expect(call.type).toBe(CallType.Video); + }); + + it("if local video", async () => { + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + + call.pushNewLocalFeed( + new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), + SDPStreamMetadataPurpose.Usermedia, + false, + ); + expect(call.type).toBe(CallType.Video); + }); + }); + + it("should correctly generate local SDPStreamMetadata", async () => { + const callPromise = call.placeCallWithCallFeeds([new CallFeed({ + client, + // @ts-ignore Mock + stream: new MockMediaStream("local_stream1", [new MockMediaStreamTrack("track_id", "audio")]), + roomId: call.roomId, + userId: client.getUserId(), + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: false, + })]); + await client.httpBackend.flush(); + await callPromise; + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + + call.pushNewLocalFeed( + new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]), + SDPStreamMetadataPurpose.Screenshare, "feed_id2", + ); + await call.setMicrophoneMuted(true); + + expect(call.getLocalSDPStreamMetadata()).toStrictEqual({ + "local_stream1": { + "purpose": SDPStreamMetadataPurpose.Usermedia, + "audio_muted": true, + "video_muted": true, + }, + "local_stream2": { + "purpose": SDPStreamMetadataPurpose.Screenshare, + "audio_muted": true, + "video_muted": false, + }, + }); + }); + + it("feed and stream getters return correctly", async () => { + const localUsermediaStream = new MockMediaStream("local_usermedia_stream_id", []); + const localScreensharingStream = new MockMediaStream("local_screensharing_stream_id", []); + const remoteUsermediaStream = new MockMediaStream("remote_usermedia_stream_id", []); + const remoteScreensharingStream = new MockMediaStream("remote_screensharing_stream_id", []); + + const callPromise = call.placeCallWithCallFeeds([ + new CallFeed({ + client, + userId: client.getUserId(), + // @ts-ignore Mock + stream: localUsermediaStream, + purpose: SDPStreamMetadataPurpose.Usermedia, + id: "local_usermedia_feed_id", + audioMuted: false, + videoMuted: false, + }), + new CallFeed({ + client, + userId: client.getUserId(), + // @ts-ignore Mock + stream: localScreensharingStream, + purpose: SDPStreamMetadataPurpose.Screenshare, + id: "local_screensharing_feed_id", + audioMuted: false, + videoMuted: false, + }), + ]); + await client.httpBackend.flush(); + await callPromise; + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + + call.updateRemoteSDPStreamMetadata({ + "remote_usermedia_stream_id": { + purpose: SDPStreamMetadataPurpose.Usermedia, + id: "remote_usermedia_feed_id", + audio_muted: false, + video_muted: false, + }, + "remote_screensharing_stream_id": { + purpose: SDPStreamMetadataPurpose.Screenshare, + id: "remote_screensharing_feed_id", + audio_muted: false, + video_muted: false, + }, + }); + call.pushRemoteFeed(remoteUsermediaStream); + call.pushRemoteFeed(remoteScreensharingStream); + + expect(call.localUsermediaFeed.stream).toBe(localUsermediaStream); + expect(call.localUsermediaStream).toBe(localUsermediaStream); + expect(call.localScreensharingFeed.stream).toBe(localScreensharingStream); + expect(call.localScreensharingStream).toBe(localScreensharingStream); + expect(call.remoteUsermediaFeed.stream).toBe(remoteUsermediaStream); + expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream); + expect(call.remoteScreensharingFeed.stream).toBe(remoteScreensharingStream); + expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream); + expect(call.hasRemoteUserMediaAudioTrack).toBe(false); + }); + + it("should end call after receiving a select event with a different party id", async () => { + const callPromise = call.initWithInvite({ + getContent: () => ({ + version: 1, + call_id: "call_id", + party_id: "remote_party_id", + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + getLocalAge: () => null, + }); + call.feeds.push(new CallFeed({ + client, + userId: "remote_user_id", + // @ts-ignore Mock + stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), + id: "remote_feed_id", + purpose: SDPStreamMetadataPurpose.Usermedia, + })); + await client.httpBackend.flush(); + await callPromise; + + const callHangupCallback = jest.fn(); + call.on(CallEvent.Hangup, callHangupCallback); + + await call.onSelectAnswerReceived({ + getContent: () => ({ + version: 1, + call_id: call.callId, + party_id: 'party_id', + selected_party_id: "different_party_id", + }), + }); + + expect(callHangupCallback).toHaveBeenCalled(); + }); + + describe("turn servers", () => { + it("should fallback if allowed", async () => { + client.client.isFallbackICEServerAllowed = () => true; + const localCall = new MatrixCall({ + client: client.client, + roomId: '!room_id', + }); + + expect((localCall as any).turnServers).toStrictEqual([{ urls: ["stun:turn.matrix.org"] }]); + }); + + it("should not fallback if not allowed", async () => { + client.client.isFallbackICEServerAllowed = () => false; + const localCall = new MatrixCall({ + client: client.client, + roomId: '!room_id', + }); + + expect((localCall as any).turnServers).toStrictEqual([]); + }); + + it("should not fallback if we supplied turn servers", async () => { + client.client.isFallbackICEServerAllowed = () => true; + const turnServers = [{ urls: ["turn.server.org"] }]; + const localCall = new MatrixCall({ + client: client.client, + roomId: '!room_id', + turnServers, + }); + + expect((localCall as any).turnServers).toStrictEqual(turnServers); + }); + }); + + it("should handle creating a data channel", async () => { + await startVoiceCall(client, call); + + const dataChannelCallback = jest.fn(); + call.on(CallEvent.DataChannel, dataChannelCallback); + + const dataChannel = call.createDataChannel("data_channel_label", { id: 123 }); + + expect(dataChannelCallback).toHaveBeenCalledWith(dataChannel); + expect(dataChannel.label).toBe("data_channel_label"); + expect(dataChannel.id).toBe(123); + }); + describe("supportsMatrixCall", () => { it("should return true when the environment is right", () => { expect(supportsMatrixCall()).toBe(true); diff --git a/spec/unit/webrtc/callFeed.spec.ts b/spec/unit/webrtc/callFeed.spec.ts new file mode 100644 index 00000000000..e8881781dd9 --- /dev/null +++ b/spec/unit/webrtc/callFeed.spec.ts @@ -0,0 +1,61 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; +import { CallFeed, CallFeedEvent } from "../../../src/webrtc/callFeed"; +import { MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; +import { TestClient } from "../../TestClient"; + +describe("CallFeed", () => { + const roomId = "room_id"; + + let client; + + beforeEach(() => { + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); + }); + + afterEach(() => { + client.stop(); + }); + + it("should handle stream replacement", () => { + const feedNewStreamCallback = jest.fn(); + const feed = new CallFeed({ + client, + roomId, + userId: "user1", + // @ts-ignore Mock + stream: new MockMediaStream("stream1"), + id: "id", + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: false, + }); + feed.on(CallFeedEvent.NewStream, feedNewStreamCallback); + + const replacementStream = new MockMediaStream("stream2"); + // @ts-ignore Mock + feed.setNewStream(replacementStream); + expect(feedNewStreamCallback).toHaveBeenCalledWith(replacementStream); + expect(feed.stream).toBe(replacementStream); + + feedNewStreamCallback.mockReset(); + + replacementStream.addTrack(new MockMediaStreamTrack("track_id", "audio")); + expect(feedNewStreamCallback).toHaveBeenCalledWith(replacementStream); + }); +}); diff --git a/src/client.ts b/src/client.ts index 3deed4e9262..a6937f94883 100644 --- a/src/client.ts +++ b/src/client.ts @@ -437,15 +437,9 @@ export enum RoomVersionStability { Unstable = "unstable", } -export interface IRoomCapability { // MSC3244 - preferred: string | null; - support: string[]; -} - export interface IRoomVersionsCapability { default: string; available: Record; - "org.matrix.msc3244.room_capabilities"?: Record; // MSC3244 } export interface ICapability { @@ -970,6 +964,7 @@ export class MatrixClient extends TypedEventEmitter !i.id_access_token); if ( invitesNeedingToken.length > 0 && - this.identityServer && - this.identityServer.getAccessToken && + this.identityServer?.getAccessToken && await this.doesServerAcceptIdentityAccessToken() ) { const identityAccessToken = await this.identityServer.getAccessToken(); @@ -8990,7 +8983,7 @@ export class MatrixClient extends TypedEventEmitter { this.replyCount = bundledRelationship.count; this._currentUserParticipated = bundledRelationship.current_user_participated; - const event = new MatrixEvent(bundledRelationship.latest_event); + const event = new MatrixEvent({ + room_id: this.rootEvent.getRoomId(), + ...bundledRelationship.latest_event, + }); this.setEventMetadata(event); event.setThread(this); this.lastEvent = event; diff --git a/src/realtime-callbacks.ts b/src/realtime-callbacks.ts index 68e41be5316..67b261515c6 100644 --- a/src/realtime-callbacks.ts +++ b/src/realtime-callbacks.ts @@ -48,20 +48,6 @@ const callbackList: { // var debuglog = logger.log.bind(logger); const debuglog = function(...params: any[]) {}; -/** - * Replace the function used by this module to get the current time. - * - * Intended for use by the unit tests. - * - * @param {function} [f] function which should return a millisecond counter - * - * @internal - */ -export function setNow(f: () => number): void { - now = f || Date.now; -} -let now = Date.now; - /** * reimplementation of window.setTimeout, which will call the callback if * the wallclock time goes past the deadline. @@ -78,7 +64,7 @@ export function setTimeout(func: (...params: any[]) => void, delayMs: number, .. delayMs = 0; } - const runAt = now() + delayMs; + const runAt = Date.now() + delayMs; const key = count++; debuglog("setTimeout: scheduling cb", key, "at", runAt, "(delay", delayMs, ")"); @@ -141,7 +127,7 @@ function scheduleRealCallback(): void { return; } - const timestamp = now(); + const timestamp = Date.now(); const delayMs = Math.min(first.runAt - timestamp, TIMER_CHECK_PERIOD_MS); debuglog("scheduleRealCallback: now:", timestamp, "delay:", delayMs); @@ -150,7 +136,7 @@ function scheduleRealCallback(): void { function runCallbacks(): void { let cb; - const timestamp = now(); + const timestamp = Date.now(); debuglog("runCallbacks: now:", timestamp); // get the list of things to call diff --git a/src/utils.ts b/src/utils.ts index 4885fb94861..6cf459097c1 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -144,23 +144,6 @@ export function checkObjectHasKeys(obj: object, keys: string[]) { } } -/** - * Checks that the given object has no extra keys other than the specified ones. - * @param {Object} obj The object to check. - * @param {string[]} allowedKeys The list of allowed key names. - * @throws If there are extra keys. - */ -export function checkObjectHasNoAdditionalKeys(obj: object, allowedKeys: string[]): void { - for (const key in obj) { - if (!obj.hasOwnProperty(key)) { - continue; - } - if (allowedKeys.indexOf(key) === -1) { - throw new Error("Unknown key: " + key); - } - } -} - /** * Deep copy the given object. The object MUST NOT have circular references and * MUST NOT have functions. @@ -283,69 +266,6 @@ export function deepSortedObjectEntries(obj: any): [string, any][] { return pairs; } -/** - * Inherit the prototype methods from one constructor into another. This is a - * port of the Node.js implementation with an Object.create polyfill. - * - * @param {function} ctor Constructor function which needs to inherit the - * prototype. - * @param {function} superCtor Constructor function to inherit prototype from. - */ -export function inherits(ctor: Function, superCtor: Function) { - // Add util.inherits from Node.js - // Source: - // https://github.com/joyent/node/blob/master/lib/util.js - // Copyright Joyent, Inc. and other Node contributors. - // - // Permission is hereby granted, free of charge, to any person obtaining a - // copy of this software and associated documentation files (the - // "Software"), to deal in the Software without restriction, including - // without limitation the rights to use, copy, modify, merge, publish, - // distribute, sublicense, and/or sell copies of the Software, and to permit - // persons to whom the Software is furnished to do so, subject to the - // following conditions: - // - // The above copyright notice and this permission notice shall be included - // in all copies or substantial portions of the Software. - // - // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS - // OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - // MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN - // NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, - // DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR - // OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE - // USE OR OTHER DEALINGS IN THE SOFTWARE. - (ctor as any).super_ = superCtor; - ctor.prototype = Object.create(superCtor.prototype, { - constructor: { - value: ctor, - enumerable: false, - writable: true, - configurable: true, - }, - }); -} - -/** - * Polyfills inheritance for prototypes by allowing different kinds of - * super types. Typically prototypes would use `SuperType.call(this, params)` - * though this doesn't always work in some environments - this function - * falls back to using `Object.assign()` to clone a constructed copy - * of the super type onto `thisArg`. - * @param {any} thisArg The child instance. Modified in place. - * @param {any} SuperType The type to act as a super instance - * @param {any} params Arguments to supply to the super type's constructor - */ -export function polyfillSuper(thisArg: any, SuperType: any, ...params: any[]) { - try { - SuperType.call(thisArg, ...params); - } catch (e) { - // fall back to Object.assign to just clone the thing - const fakeSuper = new SuperType(...params); - Object.assign(thisArg, fakeSuper); - } -} - /** * Returns whether the given value is a finite number without type-coercion * diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index c23133ea493..ad97246c8ba 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -246,7 +246,7 @@ export enum CallErrorCode { /** * The version field that we set in m.call.* events */ -const VOIP_PROTO_VERSION = 1; +const VOIP_PROTO_VERSION = "1"; /** The fallback ICE server to use for STUN or TURN protocols. */ const FALLBACK_ICE_SERVER = 'stun:turn.matrix.org'; @@ -804,9 +804,9 @@ export class MatrixCall extends TypedEventEmitter { stats.push(item); - } + }); return stats; } @@ -1019,7 +1019,7 @@ export class MatrixCall extends TypedEventEmitter= 1) || reason !== CallErrorCode.UserHangup) { + if ((this.opponentVersion && this.opponentVersion !== 0) || reason !== CallErrorCode.UserHangup) { content["reason"] = reason; } this.sendVoipEvent(EventType.CallHangup, content); @@ -1035,7 +1035,7 @@ export class MatrixCall extends TypedEventEmitter=2.2.7 <3" ace-builds@^1.4.13: - version "1.5.3" - resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.5.3.tgz#05f81d3464a9ea19696e5e6fd0f924d37dab442f" - integrity sha512-WN5BKR2aTSuBmisO8jo3Fytk6sOmJGki82v/Boeic81IgYN8pFHNkXq2anDF0XkmfDWMqLbRoW9sjc/GtKzQbQ== + version "1.7.1" + resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.7.1.tgz#be796fbd98610dda5e138aed98d309cac2ab0872" + integrity sha512-1mcbP5kXvr729sJ9dA/8tul0pjuvKbma0LF/ZMRwPEwjoNWNpe/x0OXpaPJo36aRpZCjRZMl5zsME3hAKTiaNw== acorn-globals@^3.0.0: version "3.1.0" @@ -1961,15 +1965,15 @@ aws4@^1.8.0: resolved "https://registry.yarnpkg.com/aws4/-/aws4-1.11.0.tgz#d61f46d83b2519250e2784daf5b09479a8b41c59" integrity sha512-xh1Rl34h6Fi1DC2WWKfxUTVqRsNnr6LsKz2+hfwDxQJWmrx8+c7ylaqBMcHfl1U1r2dsifOvKX3LQuLNZ+XSvA== -babel-jest@^28.0.0, babel-jest@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-28.1.0.tgz#95a67f8e2e7c0042e7b3ad3951b8af41a533b5ea" - integrity sha512-zNKk0yhDZ6QUwfxh9k07GII6siNGMJWVUU49gmFj5gfdqDKLqa2RArXOF2CODp4Dr7dLxN2cvAV+667dGJ4b4w== +babel-jest@^28.0.0, babel-jest@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/babel-jest/-/babel-jest-28.1.2.tgz#2b37fb81439f14d34d8b2cc4a4bd7efabf9acbfe" + integrity sha512-pfmoo6sh4L/+5/G2OOfQrGJgvH7fTa1oChnuYH2G/6gA+JwDvO8PELwvwnofKBMNrQsam0Wy/Rw+QSrBNewq2Q== dependencies: - "@jest/transform" "^28.1.0" + "@jest/transform" "^28.1.2" "@types/babel__core" "^7.1.14" babel-plugin-istanbul "^6.1.1" - babel-preset-jest "^28.0.2" + babel-preset-jest "^28.1.1" chalk "^4.0.0" graceful-fs "^4.2.9" slash "^3.0.0" @@ -1992,17 +1996,17 @@ babel-plugin-istanbul@^6.1.1: istanbul-lib-instrument "^5.0.4" test-exclude "^6.0.0" -babel-plugin-jest-hoist@^28.0.2: - version "28.0.2" - resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-28.0.2.tgz#9307d03a633be6fc4b1a6bc5c3a87e22bd01dd3b" - integrity sha512-Kizhn/ZL+68ZQHxSnHyuvJv8IchXD62KQxV77TBDV/xoBFBOfgRAk97GNs6hXdTTCiVES9nB2I6+7MXXrk5llQ== +babel-plugin-jest-hoist@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/babel-plugin-jest-hoist/-/babel-plugin-jest-hoist-28.1.1.tgz#5e055cdcc47894f28341f87f5e35aad2df680b11" + integrity sha512-NovGCy5Hn25uMJSAU8FaHqzs13cFoOI4lhIujiepssjCKRsAo3TA734RDWSGxuFTsUJXerYOqQQodlxgmtqbzw== dependencies: "@babel/template" "^7.3.3" "@babel/types" "^7.3.3" "@types/babel__core" "^7.1.14" "@types/babel__traverse" "^7.0.6" -babel-plugin-polyfill-corejs2@^0.3.0: +babel-plugin-polyfill-corejs2@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs2/-/babel-plugin-polyfill-corejs2-0.3.1.tgz#440f1b70ccfaabc6b676d196239b138f8a2cfba5" integrity sha512-v7/T6EQcNfVLfcN2X8Lulb7DjprieyLWJK/zOWH5DUYcAgex9sP3h25Q+DLsX9TloXe3y1O8l2q2Jv9q8UVB9w== @@ -2011,7 +2015,7 @@ babel-plugin-polyfill-corejs2@^0.3.0: "@babel/helper-define-polyfill-provider" "^0.3.1" semver "^6.1.1" -babel-plugin-polyfill-corejs3@^0.5.0: +babel-plugin-polyfill-corejs3@^0.5.2: version "0.5.2" resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-corejs3/-/babel-plugin-polyfill-corejs3-0.5.2.tgz#aabe4b2fa04a6e038b688c5e55d44e78cd3a5f72" integrity sha512-G3uJih0XWiID451fpeFaYGVuxHEjzKTHtc9uGFEjR6hHrvNzeS/PX+LLLcetJcytsB5m4j+K3o/EpXJNb/5IEQ== @@ -2019,7 +2023,7 @@ babel-plugin-polyfill-corejs3@^0.5.0: "@babel/helper-define-polyfill-provider" "^0.3.1" core-js-compat "^3.21.0" -babel-plugin-polyfill-regenerator@^0.3.0: +babel-plugin-polyfill-regenerator@^0.3.1: version "0.3.1" resolved "https://registry.yarnpkg.com/babel-plugin-polyfill-regenerator/-/babel-plugin-polyfill-regenerator-0.3.1.tgz#2c0678ea47c75c8cc2fbb1852278d8fb68233990" integrity sha512-Y2B06tvgHYt1x0yz17jGkGeeMr5FeKUu+ASJ+N6nB5lQ8Dapfg42i0OVrf8PNGJ3zKL4A23snMi1IRwrqqND7A== @@ -2044,12 +2048,12 @@ babel-preset-current-node-syntax@^1.0.0: "@babel/plugin-syntax-optional-chaining" "^7.8.3" "@babel/plugin-syntax-top-level-await" "^7.8.3" -babel-preset-jest@^28.0.2: - version "28.0.2" - resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-28.0.2.tgz#d8210fe4e46c1017e9fa13d7794b166e93aa9f89" - integrity sha512-sYzXIdgIXXroJTFeB3S6sNDWtlJ2dllCdTEsnZ65ACrMojj3hVNFRmnJ1HZtomGi+Be7aqpY/HJ92fr8OhKVkQ== +babel-preset-jest@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/babel-preset-jest/-/babel-preset-jest-28.1.1.tgz#5b6e5e69f963eb2d70f739c607b8f723c0ee75e4" + integrity sha512-FCq9Oud0ReTeWtcneYf/48981aTfXYuB9gbU4rBNNJVBSQ6ssv7E6v/qvbBxtOWwZFXjLZwpg+W3q7J6vhH25g== dependencies: - babel-plugin-jest-hoist "^28.0.2" + babel-plugin-jest-hoist "^28.1.1" babel-preset-current-node-syntax "^1.0.0" babel-runtime@^6.26.0: @@ -2312,16 +2316,15 @@ browserify@^17.0.0: vm-browserify "^1.0.0" xtend "^4.0.0" -browserslist@^4.20.2, browserslist@^4.20.3: - version "4.20.3" - resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.20.3.tgz#eb7572f49ec430e054f56d52ff0ebe9be915f8bf" - integrity sha512-NBhymBQl1zM0Y5dQT/O+xiLP9/rzOIQdKM/eMJBAq7yBgaB6krIYLGejrwVYnSHZdqjscB1SPuAjHwxjvN6Wdg== +browserslist@^4.20.2, browserslist@^4.21.0: + version "4.21.1" + resolved "https://registry.yarnpkg.com/browserslist/-/browserslist-4.21.1.tgz#c9b9b0a54c7607e8dc3e01a0d311727188011a00" + integrity sha512-Nq8MFCSrnJXSc88yliwlzQe3qNe3VntIjhsArW9IJOEPSHNx23FalwApUVbzAWABLhYJJ7y8AynWI/XM8OdfjQ== dependencies: - caniuse-lite "^1.0.30001332" - electron-to-chromium "^1.4.118" - escalade "^3.1.1" - node-releases "^2.0.3" - picocolors "^1.0.0" + caniuse-lite "^1.0.30001359" + electron-to-chromium "^1.4.172" + node-releases "^2.0.5" + update-browserslist-db "^1.0.4" bs58@^4.0.1: version "4.0.1" @@ -2411,10 +2414,10 @@ camelcase@^6.2.0: resolved "https://registry.yarnpkg.com/camelcase/-/camelcase-6.3.0.tgz#5685b95eb209ac9c0c177467778c9c84df58ba9a" integrity sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA== -caniuse-lite@^1.0.30001332: - version "1.0.30001344" - resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001344.tgz#8a1e7fdc4db9c2ec79a05e9fd68eb93a761888bb" - integrity sha512-0ZFjnlCaXNOAYcV7i+TtdKBp0L/3XEU2MF/x6Du1lrh+SRX4IfzIVL4HNJg5pB2PmFb8rszIGyOvsZnqqRoc2g== +caniuse-lite@^1.0.30001359: + version "1.0.30001363" + resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001363.tgz#26bec2d606924ba318235944e1193304ea7c4f15" + integrity sha512-HpQhpzTGGPVMnCjIomjt+jvyUu8vNFo3TaDiZ/RcoTrlOq/5+tC8zHdsbgFB6MxmaY+jCpsH09aD80Bb4Ow3Sg== caseless@~0.12.0: version "0.12.0" @@ -2481,9 +2484,9 @@ chokidar@^3.4.0: fsevents "~2.3.2" ci-info@^3.2.0: - version "3.3.1" - resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.3.1.tgz#58331f6f472a25fe3a50a351ae3052936c2c7f32" - integrity sha512-SXgeMX9VwDe7iFFaEWkA5AstuER9YKqy4EhHqr4DVqkwmD9rpVimkMKWHdjn30Ja45txyjhSn63lVX69eVCckg== + version "3.3.2" + resolved "https://registry.yarnpkg.com/ci-info/-/ci-info-3.3.2.tgz#6d2967ffa407466481c6c90b6e16b3098f080128" + integrity sha512-xmDt/QIAdeZ9+nfdPsaBCpMvHNLFiLdjj59qjqn+6iPe6YmHGQ35sBnQ8uslRBXFmXkiZQOJRjvQeoGppoTjjg== cipher-base@^1.0.0, cipher-base@^1.0.1, cipher-base@^1.0.3: version "1.0.4" @@ -2662,11 +2665,11 @@ convert-source-map@~1.1.0: integrity sha512-Y8L5rp6jo+g9VEPgvqNfEopjTR4OTYct8lXlS8iVQdmnjDvbdbzYe9rjtFCB9egC86JoNCU61WRY+ScjkZpnIg== core-js-compat@^3.21.0, core-js-compat@^3.22.1: - version "3.22.7" - resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.22.7.tgz#8359eb66ecbf726dd0cfced8e48d5e73f3224239" - integrity sha512-uI9DAQKKiiE/mclIC5g4AjRpio27g+VMRhe6rQoz+q4Wm4L6A/fJhiLtBw+sfOpDG9wZ3O0pxIw7GbfOlBgjOA== + version "3.23.3" + resolved "https://registry.yarnpkg.com/core-js-compat/-/core-js-compat-3.23.3.tgz#7d8503185be76bb6d8d592c291a4457a8e440aa9" + integrity sha512-WSzUs2h2vvmKsacLHNTdpyOC9k43AEhcGoFlVgCY4L7aw98oSBKtPL6vD0/TqZjRWRQYdDSLkzZIni4Crbbiqw== dependencies: - browserslist "^4.20.3" + browserslist "^4.21.0" semver "7.0.0" core-js@^2.4.0: @@ -2675,9 +2678,9 @@ core-js@^2.4.0: integrity sha512-Kb2wC0fvsWfQrgk8HU5lW6U/Lcs8+9aaYcy4ZFc6DDlo4nZ7n70dEgE5rtR0oG6ufKDUnrwfWL1mXR5ljDatrQ== core-js@^3.4: - version "3.22.7" - resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.22.7.tgz#8d6c37f630f6139b8732d10f2c114c3f1d00024f" - integrity sha512-Jt8SReuDKVNZnZEzyEQT5eK6T2RRCXkfTq7Lo09kpm+fHjgGewSbNjV+Wt4yZMhPDdzz2x1ulI5z/w4nxpBseg== + version "3.23.3" + resolved "https://registry.yarnpkg.com/core-js/-/core-js-3.23.3.tgz#3b977612b15da6da0c9cc4aec487e8d24f371112" + integrity sha512-oAKwkj9xcWNBAvGbT//WiCdOMpb9XQG92/Fe3ABFM/R16BsHgePG00mFOgKf7IsCtfj8tA1kHtf/VwErhriz5Q== core-util-is@1.0.2: version "1.0.2" @@ -2872,15 +2875,10 @@ diff-match-patch@^1.0.5: resolved "https://registry.yarnpkg.com/diff-match-patch/-/diff-match-patch-1.0.5.tgz#abb584d5f10cd1196dfc55aa03701592ae3f7b37" integrity sha512-IayShXAgj/QMXgB0IWmKx+rOPuGMhqm5w6jvFxmVenXKIzRqTAAsbBPT3kWQeGANj3jGgvcvv4yK6SxqYmikgw== -diff-sequences@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-27.5.1.tgz#eaecc0d327fd68c8d9672a1e64ab8dccb2ef5327" - integrity sha512-k1gCAXAsNgLwEL+Y8Wvl+M6oEFj5bgazfZULpS5CneoPPXRaCCW7dm+q21Ky2VEE5X+VeRDBVg1Pcvvsr4TtNQ== - -diff-sequences@^28.0.2: - version "28.0.2" - resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-28.0.2.tgz#40f8d4ffa081acbd8902ba35c798458d0ff1af41" - integrity sha512-YtEoNynLDFCRznv/XDalsKGSZDoj0U5kLnXvY0JSq3nBboRrZXjD81+eSiwi+nzcZDwedMmcowcxNwwgFW23mQ== +diff-sequences@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-28.1.1.tgz#9989dc731266dc2903457a70e996f3a041913ac6" + integrity sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw== diffie-hellman@^5.0.0: version "5.0.3" @@ -2949,10 +2947,10 @@ ecc-jsbn@~0.1.1: jsbn "~0.1.0" safer-buffer "^2.1.0" -electron-to-chromium@^1.4.118: - version "1.4.142" - resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.142.tgz#70cc8871f7c0122b29256089989e67cee637b40d" - integrity sha512-ea8Q1YX0JRp4GylOmX4gFHIizi0j9GfRW4EkaHnkZp0agRCBB4ZGeCv17IEzIvBkiYVwfoKVhKZJbTfqCRdQdg== +electron-to-chromium@^1.4.172: + version "1.4.177" + resolved "https://registry.yarnpkg.com/electron-to-chromium/-/electron-to-chromium-1.4.177.tgz#b6a4436eb788ca732556cd69f384b8a3c82118c5" + integrity sha512-FYPir3NSBEGexSZUEeht81oVhHfLFl6mhUKSkjHN/iB/TwEIt/WHQrqVGfTLN5gQxwJCQkIJBe05eOXjI7omgg== elliptic@^6.5.3: version "6.5.4" @@ -3173,10 +3171,10 @@ eslint-visitor-keys@^3.3.0: resolved "https://registry.yarnpkg.com/eslint-visitor-keys/-/eslint-visitor-keys-3.3.0.tgz#f6480fa6b1f30efe2d1968aa8ac745b862469826" integrity sha512-mQ+suqKJVyeuwGYHAdjMFqjCyfl8+Ldnxuyp3ldiMBFKkvytrXUZWaiPCEav8qDHKty44bD+qV1IP4T+w+xXRA== -eslint@8.16.0: - version "8.16.0" - resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.16.0.tgz#6d936e2d524599f2a86c708483b4c372c5d3bbae" - integrity sha512-MBndsoXY/PeVTDJeWsYj7kLZ5hQpJOfMYLsF6LicLHQWbRDG19lK5jOix4DPl8yY4SUFcE3txy86OzFLWT+yoA== +eslint@8.18.0: + version "8.18.0" + resolved "https://registry.yarnpkg.com/eslint/-/eslint-8.18.0.tgz#78d565d16c993d0b73968c523c0446b13da784fd" + integrity sha512-As1EfFMVk7Xc6/CvhssHUjsAQSkpfXvUGMFC3ce8JDe6WvqCgRrLOBQbVpsBFr1X1V+RACOadnzVvcUS5ni2bA== dependencies: "@eslint/eslintrc" "^1.3.0" "@humanwhocodes/config-array" "^0.9.2" @@ -3317,16 +3315,16 @@ exorcist@^1.0.1: mkdirp "~0.5.1" mold-source-map "~0.4.0" -expect@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/expect/-/expect-28.1.0.tgz#10e8da64c0850eb8c39a480199f14537f46e8360" - integrity sha512-qFXKl8Pmxk8TBGfaFKRtcQjfXEnKAs+dmlxdwvukJZorwrAabT7M3h8oLOG01I2utEhkmUTi17CHaPBovZsKdw== +expect@^28.1.0, expect@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/expect/-/expect-28.1.1.tgz#ca6fff65f6517cf7220c2e805a49c19aea30b420" + integrity sha512-/AANEwGL0tWBwzLNOvO0yUdy2D52jVdNXppOqswC49sxMN2cPWsGCQdzuIf9tj6hHoBQzNvx75JUYuQAckPo3w== dependencies: - "@jest/expect-utils" "^28.1.0" + "@jest/expect-utils" "^28.1.1" jest-get-type "^28.0.2" - jest-matcher-utils "^28.1.0" - jest-message-util "^28.1.0" - jest-util "^28.1.0" + jest-matcher-utils "^28.1.1" + jest-message-util "^28.1.1" + jest-util "^28.1.1" ext@^1.1.2: version "1.6.0" @@ -3351,9 +3349,9 @@ extsprintf@^1.2.0: integrity sha512-Wrk35e8ydCKDj/ArClo1VrPVmN8zph5V4AtHwIuHhvMXsKf73UT3BOD+azBIW+3wOJ4FhEH7zyaJCFvChjYvMA== fake-indexeddb@^3.1.2: - version "3.1.7" - resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-3.1.7.tgz#d9efbeade113c15efbe862e4598a4b0a1797ed9f" - integrity sha512-CUGeCzCOVjmeKi2C0pcvSh6NDU6uQIaS+7YyR++tO/atJJujkBYVhDvfePdz/U8bD33BMVWirsr1MKczfAqbjA== + version "3.1.8" + resolved "https://registry.yarnpkg.com/fake-indexeddb/-/fake-indexeddb-3.1.8.tgz#229e3cff6fa7355aebb3f147b908d2efa4605d70" + integrity sha512-7umIgcdnDfNcjw0ZaoD6yR2BflngKmPsyzZC+sV2fdttwz5bH6B6CCaNzzD+MURfRg8pvr/aL0trfNx65FLiDg== dependencies: realistic-structured-clone "^2.0.1" @@ -3464,9 +3462,9 @@ flat-cache@^3.0.4: rimraf "^3.0.2" flatted@^3.1.0: - version "3.2.5" - resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" - integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== + version "3.2.6" + resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.6.tgz#022e9218c637f9f3fc9c35ab9c9193f05add60b2" + integrity sha512-0sQoMh9s0BYsm+12Huy/rkKxVu4R1+r96YX5cG44rHV0pQ6iC3Q+mkoMFaGWObMFYQxCVT+ssG1ksneA2MI9KQ== for-each@^0.3.3: version "0.3.3" @@ -3562,13 +3560,13 @@ get-caller-file@^2.0.5: integrity sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg== get-intrinsic@^1.0.2, get-intrinsic@^1.1.0, get-intrinsic@^1.1.1: - version "1.1.1" - resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.1.tgz#15f59f376f855c446963948f0d24cd3637b4abc6" - integrity sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q== + version "1.1.2" + resolved "https://registry.yarnpkg.com/get-intrinsic/-/get-intrinsic-1.1.2.tgz#336975123e05ad0b7ba41f152ee4aadbea6cf598" + integrity sha512-Jfm3OyCxHh9DJyc28qGk+JmfkpO41A4XkneDSujN9MDXrm4oDKdHvndhZ2dN94+ERNfkYJWDclW6k2L/ZGHjXA== dependencies: function-bind "^1.1.1" has "^1.0.3" - has-symbols "^1.0.1" + has-symbols "^1.0.3" get-package-type@^0.1.0: version "0.1.0" @@ -3726,7 +3724,7 @@ hash.js@^1.0.0, hash.js@^1.0.3: inherits "^2.0.3" minimalistic-assert "^1.0.1" -he@^1.1.0: +he@^1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/he/-/he-1.2.0.tgz#84ae65fa7eafb165fddb61566ae14baf05664f0f" integrity sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw== @@ -3895,7 +3893,7 @@ is-callable@^1.1.3, is-callable@^1.1.4, is-callable@^1.2.4: resolved "https://registry.yarnpkg.com/is-callable/-/is-callable-1.2.4.tgz#47301d58dd0259407865547853df6d61fe471945" integrity sha512-nsuwtxZfMX67Oryl9LCQ+upnC0Z0BgpwntpS89m1H/TLF0zNfzfLMV/9Wa/6MZsj0acpEjAO0KF1xT6ZdLl95w== -is-core-module@^2.8.1: +is-core-module@^2.8.1, is-core-module@^2.9.0: version "2.9.0" resolved "https://registry.yarnpkg.com/is-core-module/-/is-core-module-2.9.0.tgz#e1c34429cd51c6dd9e09e0799e396e27b19a9c69" integrity sha512-+5FPy5PnwmO3lvfMb0AsoPaBG+5KHUI0wYFXOtYPnVVVspTFUuMZNfNaNVRt3FZadstu2c8x23vykRW/NBoU6A== @@ -4117,210 +4115,185 @@ jest-changed-files@^28.0.2: execa "^5.0.0" throat "^6.0.1" -jest-circus@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-28.1.0.tgz#e229f590911bd54d60efaf076f7acd9360296dae" - integrity sha512-rNYfqfLC0L0zQKRKsg4n4J+W1A2fbyGH7Ss/kDIocp9KXD9iaL111glsLu7+Z7FHuZxwzInMDXq+N1ZIBkI/TQ== +jest-circus@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-circus/-/jest-circus-28.1.2.tgz#0d5a5623eccb244efe87d1edc365696e4fcf80ce" + integrity sha512-E2vdPIJG5/69EMpslFhaA46WkcrN74LI5V/cSJ59L7uS8UNoXbzTxmwhpi9XrIL3zqvMt5T0pl5k2l2u2GwBNQ== dependencies: - "@jest/environment" "^28.1.0" - "@jest/expect" "^28.1.0" - "@jest/test-result" "^28.1.0" - "@jest/types" "^28.1.0" + "@jest/environment" "^28.1.2" + "@jest/expect" "^28.1.2" + "@jest/test-result" "^28.1.1" + "@jest/types" "^28.1.1" "@types/node" "*" chalk "^4.0.0" co "^4.6.0" dedent "^0.7.0" is-generator-fn "^2.0.0" - jest-each "^28.1.0" - jest-matcher-utils "^28.1.0" - jest-message-util "^28.1.0" - jest-runtime "^28.1.0" - jest-snapshot "^28.1.0" - jest-util "^28.1.0" - pretty-format "^28.1.0" + jest-each "^28.1.1" + jest-matcher-utils "^28.1.1" + jest-message-util "^28.1.1" + jest-runtime "^28.1.2" + jest-snapshot "^28.1.2" + jest-util "^28.1.1" + pretty-format "^28.1.1" slash "^3.0.0" stack-utils "^2.0.3" throat "^6.0.1" -jest-cli@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-28.1.0.tgz#cd1d8adb9630102d5ba04a22895f63decdd7ac1f" - integrity sha512-fDJRt6WPRriHrBsvvgb93OxgajHHsJbk4jZxiPqmZbMDRcHskfJBBfTyjFko0jjfprP544hOktdSi9HVgl4VUQ== +jest-cli@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-cli/-/jest-cli-28.1.2.tgz#b89012e5bad14135e71b1628b85475d3773a1bbc" + integrity sha512-l6eoi5Do/IJUXAFL9qRmDiFpBeEJAnjJb1dcd9i/VWfVWbp3mJhuH50dNtX67Ali4Ecvt4eBkWb4hXhPHkAZTw== dependencies: - "@jest/core" "^28.1.0" - "@jest/test-result" "^28.1.0" - "@jest/types" "^28.1.0" + "@jest/core" "^28.1.2" + "@jest/test-result" "^28.1.1" + "@jest/types" "^28.1.1" chalk "^4.0.0" exit "^0.1.2" graceful-fs "^4.2.9" import-local "^3.0.2" - jest-config "^28.1.0" - jest-util "^28.1.0" - jest-validate "^28.1.0" + jest-config "^28.1.2" + jest-util "^28.1.1" + jest-validate "^28.1.1" prompts "^2.0.1" yargs "^17.3.1" -jest-config@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-28.1.0.tgz#fca22ca0760e746fe1ce1f9406f6b307ab818501" - integrity sha512-aOV80E9LeWrmflp7hfZNn/zGA4QKv/xsn2w8QCBP0t0+YqObuCWTSgNbHJ0j9YsTuCO08ZR/wsvlxqqHX20iUA== +jest-config@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-config/-/jest-config-28.1.2.tgz#ba00ad30caf62286c86e7c1099e915218a0ac8c6" + integrity sha512-g6EfeRqddVbjPVBVY4JWpUY4IvQoFRIZcv4V36QkqzE0IGhEC/VkugFeBMAeUE7PRgC8KJF0yvJNDeQRbamEVA== dependencies: "@babel/core" "^7.11.6" - "@jest/test-sequencer" "^28.1.0" - "@jest/types" "^28.1.0" - babel-jest "^28.1.0" + "@jest/test-sequencer" "^28.1.1" + "@jest/types" "^28.1.1" + babel-jest "^28.1.2" chalk "^4.0.0" ci-info "^3.2.0" deepmerge "^4.2.2" glob "^7.1.3" graceful-fs "^4.2.9" - jest-circus "^28.1.0" - jest-environment-node "^28.1.0" + jest-circus "^28.1.2" + jest-environment-node "^28.1.2" jest-get-type "^28.0.2" jest-regex-util "^28.0.2" - jest-resolve "^28.1.0" - jest-runner "^28.1.0" - jest-util "^28.1.0" - jest-validate "^28.1.0" + jest-resolve "^28.1.1" + jest-runner "^28.1.2" + jest-util "^28.1.1" + jest-validate "^28.1.1" micromatch "^4.0.4" parse-json "^5.2.0" - pretty-format "^28.1.0" + pretty-format "^28.1.1" slash "^3.0.0" strip-json-comments "^3.1.1" -jest-diff@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-27.5.1.tgz#a07f5011ac9e6643cf8a95a462b7b1ecf6680def" - integrity sha512-m0NvkX55LDt9T4mctTEgnZk3fmEg3NRYutvMPWM/0iPnkFj2wIeF45O1718cMSOFO1vINkqmxqD8vE37uTEbqw== +jest-diff@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-28.1.1.tgz#1a3eedfd81ae79810931c63a1d0f201b9120106c" + integrity sha512-/MUUxeR2fHbqHoMMiffe/Afm+U8U4olFRJ0hiVG2lZatPJcnGxx292ustVu7bULhjV65IYMxRdploAKLbcrsyg== dependencies: chalk "^4.0.0" - diff-sequences "^27.5.1" - jest-get-type "^27.5.1" - pretty-format "^27.5.1" - -jest-diff@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-28.1.0.tgz#77686fef899ec1873dbfbf9330e37dd429703269" - integrity sha512-8eFd3U3OkIKRtlasXfiAQfbovgFgRDb0Ngcs2E+FMeBZ4rUezqIaGjuyggJBp+llosQXNEWofk/Sz4Hr5gMUhA== - dependencies: - chalk "^4.0.0" - diff-sequences "^28.0.2" + diff-sequences "^28.1.1" jest-get-type "^28.0.2" - pretty-format "^28.1.0" + pretty-format "^28.1.1" -jest-docblock@^28.0.2: - version "28.0.2" - resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-28.0.2.tgz#3cab8abea53275c9d670cdca814fc89fba1298c2" - integrity sha512-FH10WWw5NxLoeSdQlJwu+MTiv60aXV/t8KEwIRGEv74WARE1cXIqh1vGdy2CraHuWOOrnzTWj/azQKqW4fO7xg== +jest-docblock@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-28.1.1.tgz#6f515c3bf841516d82ecd57a62eed9204c2f42a8" + integrity sha512-3wayBVNiOYx0cwAbl9rwm5kKFP8yHH3d/fkEaL02NPTkDojPtheGB7HZSFY4wzX+DxyrvhXz0KSCVksmCknCuA== dependencies: detect-newline "^3.0.0" -jest-each@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-28.1.0.tgz#54ae66d6a0a5b1913e9a87588d26c2687c39458b" - integrity sha512-a/XX02xF5NTspceMpHujmOexvJ4GftpYXqr6HhhmKmExtMXsyIN/fvanQlt/BcgFoRKN4OCXxLQKth9/n6OPFg== +jest-each@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-28.1.1.tgz#ba5238dacf4f31d9fe23ddc2c44c01e7c23885c4" + integrity sha512-A042rqh17ZvEhRceDMi784ppoXR7MWGDEKTXEZXb4svt0eShMZvijGxzKsx+yIjeE8QYmHPrnHiTSQVhN4nqaw== dependencies: - "@jest/types" "^28.1.0" + "@jest/types" "^28.1.1" chalk "^4.0.0" jest-get-type "^28.0.2" - jest-util "^28.1.0" - pretty-format "^28.1.0" + jest-util "^28.1.1" + pretty-format "^28.1.1" -jest-environment-node@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-28.1.0.tgz#6ed2150aa31babba0c488c5b4f4d813a585c68e6" - integrity sha512-gBLZNiyrPw9CSMlTXF1yJhaBgWDPVvH0Pq6bOEwGMXaYNzhzhw2kA/OijNF8egbCgDS0/veRv97249x2CX+udQ== +jest-environment-node@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-28.1.2.tgz#3e2eb47f6d173b0648d5f7c717cb1c26651d5c8a" + integrity sha512-oYsZz9Qw27XKmOgTtnl0jW7VplJkN2oeof+SwAwKFQacq3CLlG9u4kTGuuLWfvu3J7bVutWlrbEQMOCL/jughw== dependencies: - "@jest/environment" "^28.1.0" - "@jest/fake-timers" "^28.1.0" - "@jest/types" "^28.1.0" + "@jest/environment" "^28.1.2" + "@jest/fake-timers" "^28.1.2" + "@jest/types" "^28.1.1" "@types/node" "*" - jest-mock "^28.1.0" - jest-util "^28.1.0" - -jest-get-type@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-27.5.1.tgz#3cd613c507b0f7ace013df407a1c1cd578bcb4f1" - integrity sha512-2KY95ksYSaK7DMBWQn6dQz3kqAf3BB64y2udeG+hv4KfSOb9qwcYQstTJc1KCbsix+wLZWZYN8t7nwX3GOBLRw== + jest-mock "^28.1.1" + jest-util "^28.1.1" jest-get-type@^28.0.2: version "28.0.2" resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-28.0.2.tgz#34622e628e4fdcd793d46db8a242227901fcf203" integrity sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA== -jest-haste-map@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-28.1.0.tgz#6c1ee2daf1c20a3e03dbd8e5b35c4d73d2349cf0" - integrity sha512-xyZ9sXV8PtKi6NCrJlmq53PyNVHzxmcfXNVvIRHpHmh1j/HChC4pwKgyjj7Z9us19JMw8PpQTJsFWOsIfT93Dw== +jest-haste-map@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-28.1.1.tgz#471685f1acd365a9394745bb97c8fc16289adca3" + integrity sha512-ZrRSE2o3Ezh7sb1KmeLEZRZ4mgufbrMwolcFHNRSjKZhpLa8TdooXOOFlSwoUzlbVs1t0l7upVRW2K7RWGHzbQ== dependencies: - "@jest/types" "^28.1.0" + "@jest/types" "^28.1.1" "@types/graceful-fs" "^4.1.3" "@types/node" "*" anymatch "^3.0.3" fb-watchman "^2.0.0" graceful-fs "^4.2.9" jest-regex-util "^28.0.2" - jest-util "^28.1.0" - jest-worker "^28.1.0" + jest-util "^28.1.1" + jest-worker "^28.1.1" micromatch "^4.0.4" - walker "^1.0.7" + walker "^1.0.8" optionalDependencies: fsevents "^2.3.2" -jest-leak-detector@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-28.1.0.tgz#b65167776a8787443214d6f3f54935a4c73c8a45" - integrity sha512-uIJDQbxwEL2AMMs2xjhZl2hw8s77c3wrPaQ9v6tXJLGaaQ+4QrNJH5vuw7hA7w/uGT/iJ42a83opAqxGHeyRIA== +jest-leak-detector@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-leak-detector/-/jest-leak-detector-28.1.1.tgz#537f37afd610a4b3f4cab15e06baf60484548efb" + integrity sha512-4jvs8V8kLbAaotE+wFR7vfUGf603cwYtFf1/PYEsyX2BAjSzj8hQSVTP6OWzseTl0xL6dyHuKs2JAks7Pfubmw== dependencies: jest-get-type "^28.0.2" - pretty-format "^28.1.0" + pretty-format "^28.1.1" jest-localstorage-mock@^2.4.6: version "2.4.21" resolved "https://registry.yarnpkg.com/jest-localstorage-mock/-/jest-localstorage-mock-2.4.21.tgz#920aa6fc8f8ab2f81e40433e48e2efdb2d81a6e0" integrity sha512-IBXPBufnfPyr4VkoQeJ+zlfWlG84P0KbL4ejcV9j3xNI0v6OWznQlH6Ke9xjSarleR11090oSeWADSUow0PmFw== -jest-matcher-utils@^27.0.0: - version "27.5.1" - resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-27.5.1.tgz#9c0cdbda8245bc22d2331729d1091308b40cf8ab" - integrity sha512-z2uTx/T6LBaCoNWNFWwChLBKYxTMcGBRjAt+2SbP929/Fflb9aa5LGma654Rz8z9HLxsrUaYzxE9T/EFIL/PAw== +jest-matcher-utils@^28.0.0, jest-matcher-utils@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-28.1.1.tgz#a7c4653c2b782ec96796eb3088060720f1e29304" + integrity sha512-NPJPRWrbmR2nAJ+1nmnfcKKzSwgfaciCCrYZzVnNoxVoyusYWIjkBMNvu0RHJe7dNj4hH3uZOPZsQA+xAYWqsw== dependencies: chalk "^4.0.0" - jest-diff "^27.5.1" - jest-get-type "^27.5.1" - pretty-format "^27.5.1" - -jest-matcher-utils@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-28.1.0.tgz#2ae398806668eeabd293c61712227cb94b250ccf" - integrity sha512-onnax0n2uTLRQFKAjC7TuaxibrPSvZgKTcSCnNUz/tOjJ9UhxNm7ZmPpoQavmTDUjXvUQ8KesWk2/VdrxIFzTQ== - dependencies: - chalk "^4.0.0" - jest-diff "^28.1.0" + jest-diff "^28.1.1" jest-get-type "^28.0.2" - pretty-format "^28.1.0" + pretty-format "^28.1.1" -jest-message-util@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-28.1.0.tgz#7e8f0b9049e948e7b94c2a52731166774ba7d0af" - integrity sha512-RpA8mpaJ/B2HphDMiDlrAZdDytkmwFqgjDZovM21F35lHGeUeCvYmm6W+sbQ0ydaLpg5bFAUuWG1cjqOl8vqrw== +jest-message-util@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-28.1.1.tgz#60aa0b475cfc08c8a9363ed2fb9108514dd9ab89" + integrity sha512-xoDOOT66fLfmTRiqkoLIU7v42mal/SqwDKvfmfiWAdJMSJiU+ozgluO7KbvoAgiwIrrGZsV7viETjc8GNrA/IQ== dependencies: "@babel/code-frame" "^7.12.13" - "@jest/types" "^28.1.0" + "@jest/types" "^28.1.1" "@types/stack-utils" "^2.0.0" chalk "^4.0.0" graceful-fs "^4.2.9" micromatch "^4.0.4" - pretty-format "^28.1.0" + pretty-format "^28.1.1" slash "^3.0.0" stack-utils "^2.0.3" -jest-mock@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-28.1.0.tgz#ccc7cc12a9b330b3182db0c651edc90d163ff73e" - integrity sha512-H7BrhggNn77WhdL7O1apG0Q/iwl0Bdd5E1ydhCJzL3oBLh/UYxAwR3EJLsBZ9XA3ZU4PA3UNw4tQjduBTCTmLw== +jest-mock@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-28.1.1.tgz#37903d269427fa1ef5b2447be874e1c62a39a371" + integrity sha512-bDCb0FjfsmKweAvE09dZT59IMkzgN0fYBH6t5S45NoJfd2DHkS3ySG2K+hucortryhO3fVuXdlxWcbtIuV/Skw== dependencies: - "@jest/types" "^28.1.0" + "@jest/types" "^28.1.1" "@types/node" "*" jest-pnp-resolver@^1.2.2: @@ -4333,111 +4306,111 @@ jest-regex-util@^28.0.2: resolved "https://registry.yarnpkg.com/jest-regex-util/-/jest-regex-util-28.0.2.tgz#afdc377a3b25fb6e80825adcf76c854e5bf47ead" integrity sha512-4s0IgyNIy0y9FK+cjoVYoxamT7Zeo7MhzqRGx7YDYmaQn1wucY9rotiGkBzzcMXTtjrCAP/f7f+E0F7+fxPNdw== -jest-resolve-dependencies@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-28.1.0.tgz#167becb8bee6e20b5ef4a3a728ec67aef6b0b79b" - integrity sha512-Ue1VYoSZquPwEvng7Uefw8RmZR+me/1kr30H2jMINjGeHgeO/JgrR6wxj2ofkJ7KSAA11W3cOrhNCbj5Dqqd9g== +jest-resolve-dependencies@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-resolve-dependencies/-/jest-resolve-dependencies-28.1.2.tgz#ca528858e0c6642d5a1dda8fc7cda10230c275bc" + integrity sha512-OXw4vbOZuyRTBi3tapWBqdyodU+T33ww5cPZORuTWkg+Y8lmsxQlVu3MWtJh6NMlKRTHQetF96yGPv01Ye7Mbg== dependencies: jest-regex-util "^28.0.2" - jest-snapshot "^28.1.0" + jest-snapshot "^28.1.2" -jest-resolve@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-28.1.0.tgz#b1f32748a6cee7d1779c7ef639c0a87078de3d35" - integrity sha512-vvfN7+tPNnnhDvISuzD1P+CRVP8cK0FHXRwPAcdDaQv4zgvwvag2n55/h5VjYcM5UJG7L4TwE5tZlzcI0X2Lhw== +jest-resolve@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-resolve/-/jest-resolve-28.1.1.tgz#bc2eaf384abdcc1aaf3ba7c50d1adf01e59095e5" + integrity sha512-/d1UbyUkf9nvsgdBildLe6LAD4DalgkgZcKd0nZ8XUGPyA/7fsnaQIlKVnDiuUXv/IeZhPEDrRJubVSulxrShA== dependencies: chalk "^4.0.0" graceful-fs "^4.2.9" - jest-haste-map "^28.1.0" + jest-haste-map "^28.1.1" jest-pnp-resolver "^1.2.2" - jest-util "^28.1.0" - jest-validate "^28.1.0" + jest-util "^28.1.1" + jest-validate "^28.1.1" resolve "^1.20.0" resolve.exports "^1.1.0" slash "^3.0.0" -jest-runner@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-28.1.0.tgz#aefe2a1e618a69baa0b24a50edc54fdd7e728eaa" - integrity sha512-FBpmuh1HB2dsLklAlRdOxNTTHKFR6G1Qmd80pVDvwbZXTriqjWqjei5DKFC1UlM732KjYcE6yuCdiF0WUCOS2w== +jest-runner@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-runner/-/jest-runner-28.1.2.tgz#f293409592a62234285a71237e38499a3554e350" + integrity sha512-6/k3DlAsAEr5VcptCMdhtRhOoYClZQmxnVMZvZ/quvPGRpN7OBQYPIC32tWSgOnbgqLXNs5RAniC+nkdFZpD4A== dependencies: - "@jest/console" "^28.1.0" - "@jest/environment" "^28.1.0" - "@jest/test-result" "^28.1.0" - "@jest/transform" "^28.1.0" - "@jest/types" "^28.1.0" + "@jest/console" "^28.1.1" + "@jest/environment" "^28.1.2" + "@jest/test-result" "^28.1.1" + "@jest/transform" "^28.1.2" + "@jest/types" "^28.1.1" "@types/node" "*" chalk "^4.0.0" emittery "^0.10.2" graceful-fs "^4.2.9" - jest-docblock "^28.0.2" - jest-environment-node "^28.1.0" - jest-haste-map "^28.1.0" - jest-leak-detector "^28.1.0" - jest-message-util "^28.1.0" - jest-resolve "^28.1.0" - jest-runtime "^28.1.0" - jest-util "^28.1.0" - jest-watcher "^28.1.0" - jest-worker "^28.1.0" + jest-docblock "^28.1.1" + jest-environment-node "^28.1.2" + jest-haste-map "^28.1.1" + jest-leak-detector "^28.1.1" + jest-message-util "^28.1.1" + jest-resolve "^28.1.1" + jest-runtime "^28.1.2" + jest-util "^28.1.1" + jest-watcher "^28.1.1" + jest-worker "^28.1.1" source-map-support "0.5.13" throat "^6.0.1" -jest-runtime@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-28.1.0.tgz#4847dcb2a4eb4b0f9eaf41306897e51fb1665631" - integrity sha512-wNYDiwhdH/TV3agaIyVF0lsJ33MhyujOe+lNTUiolqKt8pchy1Hq4+tDMGbtD5P/oNLA3zYrpx73T9dMTOCAcg== - dependencies: - "@jest/environment" "^28.1.0" - "@jest/fake-timers" "^28.1.0" - "@jest/globals" "^28.1.0" - "@jest/source-map" "^28.0.2" - "@jest/test-result" "^28.1.0" - "@jest/transform" "^28.1.0" - "@jest/types" "^28.1.0" +jest-runtime@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-runtime/-/jest-runtime-28.1.2.tgz#d68f34f814a848555a345ceda23289f14d59a688" + integrity sha512-i4w93OsWzLOeMXSi9epmakb2+3z0AchZtUQVF1hesBmcQQy4vtaql5YdVe9KexdJaVRyPDw8DoBR0j3lYsZVYw== + dependencies: + "@jest/environment" "^28.1.2" + "@jest/fake-timers" "^28.1.2" + "@jest/globals" "^28.1.2" + "@jest/source-map" "^28.1.2" + "@jest/test-result" "^28.1.1" + "@jest/transform" "^28.1.2" + "@jest/types" "^28.1.1" chalk "^4.0.0" cjs-module-lexer "^1.0.0" collect-v8-coverage "^1.0.0" execa "^5.0.0" glob "^7.1.3" graceful-fs "^4.2.9" - jest-haste-map "^28.1.0" - jest-message-util "^28.1.0" - jest-mock "^28.1.0" + jest-haste-map "^28.1.1" + jest-message-util "^28.1.1" + jest-mock "^28.1.1" jest-regex-util "^28.0.2" - jest-resolve "^28.1.0" - jest-snapshot "^28.1.0" - jest-util "^28.1.0" + jest-resolve "^28.1.1" + jest-snapshot "^28.1.2" + jest-util "^28.1.1" slash "^3.0.0" strip-bom "^4.0.0" -jest-snapshot@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-28.1.0.tgz#4b74fa8816707dd10fe9d551c2c258e5a67b53b6" - integrity sha512-ex49M2ZrZsUyQLpLGxQtDbahvgBjlLPgklkqGM0hq/F7W/f8DyqZxVHjdy19QKBm4O93eDp+H5S23EiTbbUmHw== +jest-snapshot@^28.1.2: + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest-snapshot/-/jest-snapshot-28.1.2.tgz#93d31b87b11b384f5946fe0767541496135f8d52" + integrity sha512-wzrieFttZYfLvrCVRJxX+jwML2YTArOUqFpCoSVy1QUapx+LlV9uLbV/mMEhYj4t7aMeE9aSQFHSvV/oNoDAMA== dependencies: "@babel/core" "^7.11.6" "@babel/generator" "^7.7.2" "@babel/plugin-syntax-typescript" "^7.7.2" "@babel/traverse" "^7.7.2" "@babel/types" "^7.3.3" - "@jest/expect-utils" "^28.1.0" - "@jest/transform" "^28.1.0" - "@jest/types" "^28.1.0" + "@jest/expect-utils" "^28.1.1" + "@jest/transform" "^28.1.2" + "@jest/types" "^28.1.1" "@types/babel__traverse" "^7.0.6" "@types/prettier" "^2.1.5" babel-preset-current-node-syntax "^1.0.0" chalk "^4.0.0" - expect "^28.1.0" + expect "^28.1.1" graceful-fs "^4.2.9" - jest-diff "^28.1.0" + jest-diff "^28.1.1" jest-get-type "^28.0.2" - jest-haste-map "^28.1.0" - jest-matcher-utils "^28.1.0" - jest-message-util "^28.1.0" - jest-util "^28.1.0" + jest-haste-map "^28.1.1" + jest-matcher-utils "^28.1.1" + jest-message-util "^28.1.1" + jest-util "^28.1.1" natural-compare "^1.4.0" - pretty-format "^28.1.0" + pretty-format "^28.1.1" semver "^7.3.5" jest-sonar-reporter@^2.0.0: @@ -4447,61 +4420,62 @@ jest-sonar-reporter@^2.0.0: dependencies: xml "^1.0.1" -jest-util@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-28.1.0.tgz#d54eb83ad77e1dd441408738c5a5043642823be5" - integrity sha512-qYdCKD77k4Hwkose2YBEqQk7PzUf/NSE+rutzceduFveQREeH6b+89Dc9+wjX9dAwHcgdx4yedGA3FQlU/qCTA== +jest-util@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-28.1.1.tgz#ff39e436a1aca397c0ab998db5a51ae2b7080d05" + integrity sha512-FktOu7ca1DZSyhPAxgxB6hfh2+9zMoJ7aEQA759Z6p45NuO8mWcqujH+UdHlCm/V6JTWwDztM2ITCzU1ijJAfw== dependencies: - "@jest/types" "^28.1.0" + "@jest/types" "^28.1.1" "@types/node" "*" chalk "^4.0.0" ci-info "^3.2.0" graceful-fs "^4.2.9" picomatch "^2.2.3" -jest-validate@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-28.1.0.tgz#8a6821f48432aba9f830c26e28226ad77b9a0e18" - integrity sha512-Lly7CJYih3vQBfjLeANGgBSBJ7pEa18cxpQfQEq2go2xyEzehnHfQTjoUia8xUv4x4J80XKFIDwJJThXtRFQXQ== +jest-validate@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-28.1.1.tgz#59b7b339b3c85b5144bd0c06ad3600f503a4acc8" + integrity sha512-Kpf6gcClqFCIZ4ti5++XemYJWUPCFUW+N2gknn+KgnDf549iLul3cBuKVe1YcWRlaF8tZV8eJCap0eECOEE3Ug== dependencies: - "@jest/types" "^28.1.0" + "@jest/types" "^28.1.1" camelcase "^6.2.0" chalk "^4.0.0" jest-get-type "^28.0.2" leven "^3.1.0" - pretty-format "^28.1.0" + pretty-format "^28.1.1" -jest-watcher@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-28.1.0.tgz#aaa7b4164a4e77eeb5f7d7b25ede5e7b4e9c9aaf" - integrity sha512-tNHMtfLE8Njcr2IRS+5rXYA4BhU90gAOwI9frTGOqd+jX0P/Au/JfRSNqsf5nUTcWdbVYuLxS1KjnzILSoR5hA== +jest-watcher@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-watcher/-/jest-watcher-28.1.1.tgz#533597fb3bfefd52b5cd115cd916cffd237fb60c" + integrity sha512-RQIpeZ8EIJMxbQrXpJQYIIlubBnB9imEHsxxE41f54ZwcqWLysL/A0ZcdMirf+XsMn3xfphVQVV4EW0/p7i7Ug== dependencies: - "@jest/test-result" "^28.1.0" - "@jest/types" "^28.1.0" + "@jest/test-result" "^28.1.1" + "@jest/types" "^28.1.1" "@types/node" "*" ansi-escapes "^4.2.1" chalk "^4.0.0" emittery "^0.10.2" - jest-util "^28.1.0" + jest-util "^28.1.1" string-length "^4.0.1" -jest-worker@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-28.1.0.tgz#ced54757a035e87591e1208253a6e3aac1a855e5" - integrity sha512-ZHwM6mNwaWBR52Snff8ZvsCTqQsvhCxP/bT1I6T6DAnb6ygkshsyLQIMxFwHpYxht0HOoqt23JlC01viI7T03A== +jest-worker@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/jest-worker/-/jest-worker-28.1.1.tgz#3480c73247171dfd01eda77200f0063ab6a3bf28" + integrity sha512-Au7slXB08C6h+xbJPp7VIb6U0XX5Kc9uel/WFc6/rcTzGiaVCBRngBExSYuXSLFPULPSYU3cJ3ybS988lNFQhQ== dependencies: "@types/node" "*" merge-stream "^2.0.0" supports-color "^8.0.0" jest@^28.0.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/jest/-/jest-28.1.0.tgz#f420e41c8f2395b9a30445a97189ebb57593d831" - integrity sha512-TZR+tHxopPhzw3c3560IJXZWLNHgpcz1Zh0w5A65vynLGNcg/5pZ+VildAd7+XGOu6jd58XMY/HNn0IkZIXVXg== + version "28.1.2" + resolved "https://registry.yarnpkg.com/jest/-/jest-28.1.2.tgz#451ff24081ce31ca00b07b60c61add13aa96f8eb" + integrity sha512-Tuf05DwLeCh2cfWCQbcz9UxldoDyiR1E9Igaei5khjonKncYdc6LDfynKCEWozK0oLE3GD+xKAo2u8x/0s6GOg== dependencies: - "@jest/core" "^28.1.0" + "@jest/core" "^28.1.2" + "@jest/types" "^28.1.1" import-local "^3.0.2" - jest-cli "^28.1.0" + jest-cli "^28.1.2" js-stringify@^1.0.1: version "1.0.2" @@ -4829,9 +4803,9 @@ markdown-it@^12.3.2: uc.micro "^1.0.5" marked@^4.0.10: - version "4.0.16" - resolved "https://registry.yarnpkg.com/marked/-/marked-4.0.16.tgz#9ec18fc1a723032eb28666100344d9428cf7a264" - integrity sha512-wahonIQ5Jnyatt2fn8KqF/nIqZM8mh3oRu2+l5EANGMhu6RFjiSG52QNE2eWzFMI94HqYSgN184NurgNG6CztA== + version "4.0.17" + resolved "https://registry.yarnpkg.com/marked/-/marked-4.0.17.tgz#1186193d85bb7882159cdcfc57d1dfccaffb3fe9" + integrity sha512-Wfk0ATOK5iPxM4ptrORkFemqroz0ZDxp5MWfYA7H/F+wO17NRWV5Ypxi6p3g2Xmw2bKeiYOl6oVnLHKxBA0VhA== matrix-events-sdk@^0.0.1-beta.7: version "0.0.1-beta.7" @@ -4839,9 +4813,9 @@ matrix-events-sdk@^0.0.1-beta.7: integrity sha512-9jl4wtWanUFSy2sr2lCjErN/oC8KTAtaeaozJtrgot1JiQcEI4Rda9OLgQ7nLKaqb4Z/QUx/fR3XpDzm5Jy1JA== matrix-mock-request@^2.0.1: - version "2.0.1" - resolved "https://registry.yarnpkg.com/matrix-mock-request/-/matrix-mock-request-2.0.1.tgz#1cf7b516f8525de8373f1d9985a4a447db80bb96" - integrity sha512-NqCSDRBUTXKY7TS5H6Fqu6oxSsWKGkyh3LTXa/T6mSGABi2zMkeqGa2r2H3rnH6waJRt5N7xn+u7vEmSpg0oBQ== + version "2.1.0" + resolved "https://registry.yarnpkg.com/matrix-mock-request/-/matrix-mock-request-2.1.0.tgz#86f5b0ef846865d0767d3a8e64f5bcd6ca94c178" + integrity sha512-Cjpl3yP6h0yu5GKG89m1XZXZlm69Kg/qHV41N/t6SrQsgcfM3Bfavqx9YrtG0UnuXGy4bBSZIe1QiWVeFPZw1A== dependencies: expect "^28.1.0" @@ -5027,7 +5001,7 @@ next-tick@1, next-tick@^1.1.0: node-dir@^0.1.10: version "0.1.17" resolved "https://registry.yarnpkg.com/node-dir/-/node-dir-0.1.17.tgz#5f5665d93351335caabef8f1c554516cf5f1e4e5" - integrity sha1-X1Zl2TNRM1yqvvjxxVRRbPXx5OU= + integrity sha512-tmPX422rYgofd4epzrNoOXiE8XFZYOcCq1vD7MAXCDO+O+zndlA2ztdKKMa+EeuBG5tHETpr4ml4RGgpqDCCAg== dependencies: minimatch "^3.0.2" @@ -5041,9 +5015,9 @@ node-fetch@^2.6.7: node-int64@^0.4.0: version "0.4.0" resolved "https://registry.yarnpkg.com/node-int64/-/node-int64-0.4.0.tgz#87a9065cdb355d3182d8f94ce11188b825c68a3b" - integrity sha1-h6kGXNs1XTGC2PlM4RGIuCXGijs= + integrity sha512-O5lz91xSOeoXP6DulyHfllpq+Eg00MWitZIbtPfoSEvqIHdl5gfcY6hYzDWnj0qD5tz52PI08u9qUvSVeUBeHw== -node-releases@^2.0.3: +node-releases@^2.0.5: version "2.0.5" resolved "https://registry.yarnpkg.com/node-releases/-/node-releases-2.0.5.tgz#280ed5bc3eba0d96ce44897d8aee478bfb3d9666" integrity sha512-U9h1NLROZTq9uE1SNffn6WuPDg8icmi3ns4rEl/oTfIle4iLjTliCzgTsbaIFMq/Xn078/lfY/BL0GWZ+psK4Q== @@ -5068,7 +5042,7 @@ oauth-sign@~0.9.0: object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" - integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= + integrity sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg== object-inspect@^1.12.0, object-inspect@^1.9.0: version "1.12.2" @@ -5102,7 +5076,7 @@ object.values@^1.1.5: once@^1.3.0, once@^1.4.0: version "1.4.0" resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" - integrity sha1-WDsap3WWHUsROsF9nFC6753Xa9E= + integrity sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w== dependencies: wrappy "1" @@ -5128,7 +5102,7 @@ optionator@^0.9.1: os-browserify@~0.3.0: version "0.3.0" resolved "https://registry.yarnpkg.com/os-browserify/-/os-browserify-0.3.0.tgz#854373c7f5c2315914fc9bfc6bd8238fdda1ec27" - integrity sha1-hUNzx/XCMVkU/Jv8a9gjj92h7Cc= + integrity sha512-gjcpUc3clBf9+210TRaDWbf+rZZZEshZ+DlXMRCeAjp0xhTrnQsKHypIy1J3d5hKdUzj69t708EHtU8P6bUn0A== p-limit@^1.1.0: version "1.3.0" @@ -5154,7 +5128,7 @@ p-limit@^3.0.2: p-locate@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-2.0.0.tgz#20a0103b222a70c8fd39cc2e580680f3dde5ec43" - integrity sha1-IKAQOyIqcMj9OcwuWAaA893l7EM= + integrity sha512-nQja7m7gSKuewoVRen45CtVfODR3crN3goVQ0DDZ9N3yHxgpkuBhZqsaiotSQRrADUrne346peY7kT3TSACykg== dependencies: p-limit "^1.1.0" @@ -5190,7 +5164,7 @@ p-retry@^4.5.0: p-try@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/p-try/-/p-try-1.0.0.tgz#cbc79cdbaf8fd4228e13f621f2b1a237c1b207b3" - integrity sha1-y8ec26+P1CKOE/Yh8rGiN8GyB7M= + integrity sha512-U1etNYuMJoIz3ZXSrrySFjsXQTWOx2/jdi86L+2pRvph/qMKL6sbcCYdH23fqsbm8TH2Gn0OybpT4eSFlCVHww== p-try@^2.0.0: version "2.2.0" @@ -5212,7 +5186,7 @@ parent-module@^1.0.0: parents@^1.0.0, parents@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/parents/-/parents-1.0.1.tgz#fedd4d2bf193a77745fe71e371d73c3307d9c751" - integrity sha1-/t1NK/GTp3dF/nHjcdc8MwfZx1E= + integrity sha512-mXKF3xkoUt5td2DoxpLmtOmZvko9VfFpwRwkKDHSNvgmpLAeBo18YDhcPbBzJq+QLCHMbGOfzia2cX4U+0v9Mg== dependencies: path-platform "~0.11.15" @@ -5230,7 +5204,7 @@ parse-asn1@^5.0.0, parse-asn1@^5.1.5: parse-json@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/parse-json/-/parse-json-2.2.0.tgz#f480f40434ef80741f8469099f8dea18f55a4dc9" - integrity sha1-9ID0BDTvgHQfhGkJn43qGPVaTck= + integrity sha512-QR/GGaKCkhwk1ePQNYDRKYZ3mwU9ypsKhB0XyFnLQdomyEqk3e8wpW3V5Jp88zbxK4n5ST1nqo+g9juTpownhQ== dependencies: error-ex "^1.2.0" @@ -5252,7 +5226,7 @@ path-browserify@^1.0.0: path-exists@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/path-exists/-/path-exists-3.0.0.tgz#ce0ebeaa5f78cb18925ea7d810d7b59b010fd515" - integrity sha1-zg6+ql94yxiSXqfYENe1mwEP1RU= + integrity sha512-bpC7GYwiDYQ4wYLe+FA8lhRjhQCMcQGuSgGGqDkg/QerRWw9CmGRT0iSOVRSZJ29NMLZgIzqaljJ63oaL4NIJQ== path-exists@^4.0.0: version "4.0.0" @@ -5262,7 +5236,7 @@ path-exists@^4.0.0: path-is-absolute@^1.0.0, path-is-absolute@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" - integrity sha1-F0uSaHNVNP+8es5r9TpanhtcX18= + integrity sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg== path-key@^3.0.0, path-key@^3.1.0: version "3.1.1" @@ -5277,7 +5251,7 @@ path-parse@^1.0.7: path-platform@~0.11.15: version "0.11.15" resolved "https://registry.yarnpkg.com/path-platform/-/path-platform-0.11.15.tgz#e864217f74c36850f0852b78dc7bf7d4a5721bf2" - integrity sha1-6GQhf3TDaFDwhSt43Hv31KVyG/I= + integrity sha512-Y30dB6rab1A/nfEKsZxmr01nUotHX0c/ZiIAsCTatEe1CmS5Pm5He7fZ195bPT7RdquoaL8lLxFCMQi/bS7IJg== path-type@^4.0.0: version "4.0.0" @@ -5298,7 +5272,7 @@ pbkdf2@^3.0.3: performance-now@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/performance-now/-/performance-now-2.1.0.tgz#6309f4e0e5fa913ec1c69307ae364b4b377c9e7b" - integrity sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns= + integrity sha512-7EAHlyLHI56VEIdK57uwHdHKIaAGbnXPiw0yWbarQZOKaKpvUIgW0jWRVLiatnM+XXlSwsanIBH/hzGMJulMow== picocolors@^1.0.0: version "1.0.0" @@ -5339,19 +5313,10 @@ prelude-ls@^1.2.1: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== -pretty-format@^27.0.0, pretty-format@^27.5.1: - version "27.5.1" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-27.5.1.tgz#2181879fdea51a7a5851fb39d920faa63f01d88e" - integrity sha512-Qb1gy5OrP5+zDf2Bvnzdl3jsTf1qXVMazbvCoKhtKqVs4/YK4ozX4gKQJJVyNe+cajNPn0KoC0MC3FUmaHWEmQ== - dependencies: - ansi-regex "^5.0.1" - ansi-styles "^5.0.0" - react-is "^17.0.1" - -pretty-format@^28.1.0: - version "28.1.0" - resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-28.1.0.tgz#8f5836c6a0dfdb834730577ec18029052191af55" - integrity sha512-79Z4wWOYCdvQkEoEuSlBhHJqWeZ8D8YRPiPctJFCtvuaClGpiwiQYSCUOE6IEKUbbFukKOTFIUAXE8N4EQTo1Q== +pretty-format@^28.0.0, pretty-format@^28.1.1: + version "28.1.1" + resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-28.1.1.tgz#f731530394e0f7fcd95aba6b43c50e02d86b95cb" + integrity sha512-wwJbVTGFHeucr5Jw2bQ9P+VYHyLdAqedFLEkdQUVaBF/eiidDwH5OpilINq4mEfhbCjLnirt6HTTDhv1HaTIQw== dependencies: "@jest/schemas" "^28.0.2" ansi-regex "^5.0.1" @@ -5371,7 +5336,7 @@ process-nextick-args@~2.0.0: process@~0.11.0: version "0.11.10" resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" - integrity sha1-czIwDoQBYb2j5podHZGn1LwW8YI= + integrity sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A== promise@^7.0.1: version "7.3.1" @@ -5400,12 +5365,12 @@ prop-types@^15.7.2: pseudomap@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" - integrity sha1-8FKijacOYYkX7wqKw0wa5aaChrM= + integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== psl@^1.1.28: - version "1.8.0" - resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" - integrity sha512-RIdOzyoavK+hA18OGGWDqUTsCLhtA7IcZ/6NCs4fFJaHBDab+pDDmDIByWFRQJq2Cd7r1OoQxBGKOaztq+hjIQ== + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== public-encrypt@^4.0.0: version "4.0.3" @@ -5527,12 +5492,12 @@ pug@^2.0.3: punycode@1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" - integrity sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0= + integrity sha512-RofWgt/7fL5wP1Y7fxE7/EmTLzQVnB0ycyibJ0OOHIlJqTNzglYFxVwETOcIoJqJmpDXJ9xImDv+Fq34F/d4Dw== punycode@^1.3.2: version "1.4.1" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.4.1.tgz#c0d5a63b2718800ad8e1eb0fa5269c84dd41845e" - integrity sha1-wNWmOycYgArY4esPpSachN1BhF4= + integrity sha512-jmYNElW7yvO7TV33CjSmvSiE2yco3bV2czu/OzDKdMNVZQWfxCblURLhf+47syQRBntjfLdd/H0egrzIG+oaFQ== punycode@^2.1.0, punycode@^2.1.1: version "2.1.1" @@ -5540,9 +5505,9 @@ punycode@^2.1.0, punycode@^2.1.1: integrity sha512-XRsRjdf+j5ml+y/6GKHPZbrF/8p2Yga0JPtdqTIY2Xe5ohJPD9saDJJLPvp9+NSBprVvevdXZybnj2cv8OEd0A== qs@^6.9.6: - version "6.10.3" - resolved "https://registry.yarnpkg.com/qs/-/qs-6.10.3.tgz#d6cde1b2ffca87b5aa57889816c5f81535e22e8e" - integrity sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ== + version "6.11.0" + resolved "https://registry.yarnpkg.com/qs/-/qs-6.11.0.tgz#fd0d963446f7a65e1367e01abd85429453f0c37a" + integrity sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q== dependencies: side-channel "^1.0.4" @@ -5554,12 +5519,12 @@ qs@~6.5.2: querystring-es3@~0.2.0: version "0.2.1" resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73" - integrity sha1-nsYfeQSYdXB9aUFFlv2Qek1xHnM= + integrity sha512-773xhDQnZBMFobEiztv8LIl70ch5MSF/jUQVlhwFyBILqq96anmoctVIYz+ZRp0qbCKATTn6ev02M3r7Ga5vqA== querystring@0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" - integrity sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA= + integrity sha512-X/xY82scca2tau62i9mDyU9K+I+djTMUsvwf7xnUX5GLvVzgJybOJf4Y6o9Zx3oJK/LSXg5tTZBjwzqVPaPO2g== queue-microtask@^1.2.2: version "1.2.3" @@ -5593,9 +5558,9 @@ react-ace@^9.5.0: prop-types "^15.7.2" react-docgen@^5.4.0: - version "5.4.1" - resolved "https://registry.yarnpkg.com/react-docgen/-/react-docgen-5.4.1.tgz#867168accce39e25095a23a922eaa90722e9d182" - integrity sha512-TZqD1aApirw86NV6tHrmDoxUn8wlinkVyutFarzbdwuhEurAzDN0y5sSj64o+BrHLPqjwpH9tunpfwgy+3Uyww== + version "5.4.3" + resolved "https://registry.yarnpkg.com/react-docgen/-/react-docgen-5.4.3.tgz#7d297f73b977d0c7611402e5fc2a168acf332b26" + integrity sha512-xlLJyOlnfr8lLEEeaDZ+X2J/KJoe6Nr9AzxnkdQWush5hz2ZSu66w6iLMOScMmxoSHWpWMn+k3v5ZiyCfcWsOA== dependencies: "@babel/core" "^7.7.5" "@babel/generator" "^7.12.11" @@ -5618,20 +5583,15 @@ react-is@^16.13.1: resolved "https://registry.yarnpkg.com/react-is/-/react-is-16.13.1.tgz#789729a4dc36de2999dc156dd6c1d9c18cea56a4" integrity sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ== -react-is@^17.0.1: - version "17.0.2" - resolved "https://registry.yarnpkg.com/react-is/-/react-is-17.0.2.tgz#e691d4a8e9c789365655539ab372762b0efb54f0" - integrity sha512-w2GsyukL62IJnlaff/nRegPQR94C/XXamvMWmSHRJ4y7Ts/4ocGRmTHvOs8PSE6pB3dWOrD/nueuU5sduBsQ4w== - react-is@^18.0.0: - version "18.1.0" - resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.1.0.tgz#61aaed3096d30eacf2a2127118b5b41387d32a67" - integrity sha512-Fl7FuabXsJnV5Q1qIOQwx/sagGF18kogb4gpfcG4gjLBWO0WDiiz1ko/ExayuxE7InyQkBLkxRFG5oxY6Uu3Kg== + version "18.2.0" + resolved "https://registry.yarnpkg.com/react-is/-/react-is-18.2.0.tgz#199431eeaaa2e09f86427efbb4f1473edb47609b" + integrity sha512-xWGDIW6x921xtzPkhiULtthJHoJvBbF3q26fzloPCK0hsvxtPVelvftw3zjbHWSkR2km9Z+4uxbDDK/6Zw9B8w== read-only-stream@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/read-only-stream/-/read-only-stream-2.0.0.tgz#2724fd6a8113d73764ac288d4386270c1dbf17f0" - integrity sha1-JyT9aoET1zdkrCiNQ4YnDB2/F/A= + integrity sha512-3ALe0bjBVZtkdWKIcThYpQCLbBMd/+Tbh2CDSrAIDO3UsZ4Xs+tnyjv2MjCOMMgBG+AsUOeuP1cgtY1INISc8w== dependencies: readable-stream "^2.0.2" @@ -5727,10 +5687,10 @@ regexpp@^3.2.0: resolved "https://registry.yarnpkg.com/regexpp/-/regexpp-3.2.0.tgz#0425a2768d8f23bad70ca4b90461fa2f1213e1b2" integrity sha512-pq2bWo9mVD43nbts2wGv17XLiNLya+GklZ8kaDLV2Z08gDCsGpnKn9BFMepvWuHCbyVvY7J5o5+BVvoQbmlJLg== -regexpu-core@^5.0.1: - version "5.0.1" - resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.0.1.tgz#c531122a7840de743dcf9c83e923b5560323ced3" - integrity sha512-CriEZlrKK9VJw/xQGJpQM5rY88BtuL8DM+AEwvcThHilbxiTAy8vq4iJnd2tqq8wLmjbGZzP7ZcKFjbGkmEFrw== +regexpu-core@^5.1.0: + version "5.1.0" + resolved "https://registry.yarnpkg.com/regexpu-core/-/regexpu-core-5.1.0.tgz#2f8504c3fd0ebe11215783a41541e21c79942c6d" + integrity sha512-bb6hk+xWd2PEOkj5It46A16zFMs2mv86Iwpdu94la4S3sJ7C973h2dHpYKwIBGaWSO7cIRJ+UX0IeMaWcO4qwA== dependencies: regenerate "^1.4.2" regenerate-unicode-properties "^10.0.1" @@ -5754,7 +5714,7 @@ regjsparser@^0.8.2: repeat-string@^1.5.2: version "1.6.1" resolved "https://registry.yarnpkg.com/repeat-string/-/repeat-string-1.6.1.tgz#8dcae470e1c88abc2d600fff4a776286da75e637" - integrity sha1-jcrkcOHIirwtYA//Sndihtp15jc= + integrity sha512-PV0dzCYDNfRi1jCDbJzpW7jNNDRuCOG/jI5ctQcGKt/clZD+YcPS3yIlWuTJMmESC8aevCFmWJy5wjAFgNqN6w== request@^2.88.2: version "2.88.2" @@ -5785,7 +5745,7 @@ request@^2.88.2: require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" - integrity sha1-jGStX9MNqxyXbiNE/+f3kqam30I= + integrity sha512-fGxEI7+wsG9xrvdjsrlmL22OMTTiHRwAMroiEeMgq8gzoLC/PQr7RsRDSTLUg/bZAZtF+TVIkHc6/4RIKrui+Q== requizzle@^0.2.3: version "0.2.3" @@ -5817,11 +5777,11 @@ resolve.exports@^1.1.0: integrity sha512-J1l+Zxxp4XK3LUDZ9m60LRJF/mAe4z6a4xyabPHk7pvK5t35dACV32iIjJDFeWZFfZlO29w6SZ67knR0tHzJtQ== resolve@^1.1.4, resolve@^1.1.6, resolve@^1.14.2, resolve@^1.17.0, resolve@^1.20.0, resolve@^1.22.0, resolve@^1.4.0: - version "1.22.0" - resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.0.tgz#5e0b8c67c15df57a89bdbabe603a002f21731198" - integrity sha512-Hhtrw0nLeSrFQ7phPp4OOcVjLPIeMnRlr5mcnVuMe7M/7eBn98A3hmFRLoFo3DLZkivSYwhRUJTyPyWAk56WLw== + version "1.22.1" + resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.22.1.tgz#27cb2ebb53f91abb49470a928bba7558066ac177" + integrity sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw== dependencies: - is-core-module "^2.8.1" + is-core-module "^2.9.0" path-parse "^1.0.7" supports-preserve-symlinks-flag "^1.0.0" @@ -5838,7 +5798,7 @@ reusify@^1.0.4: right-align@^0.1.1: version "0.1.3" resolved "https://registry.yarnpkg.com/right-align/-/right-align-0.1.3.tgz#61339b722fe6a3515689210d24e14c96148613ef" - integrity sha1-YTObci/mo1FWiSENJOFMlhSGE+8= + integrity sha512-yqINtL/G7vs2v+dFIZmFUDbnVyFUJFKd6gK22Kgo6R4jfJGFtisKyncWDDULgjfqf4ASQuIQyjJ7XZ+3aWpsAg== dependencies: align-text "^0.1.1" @@ -6003,12 +5963,12 @@ source-map@^0.6.0, source-map@^0.6.1, source-map@~0.6.0, source-map@~0.6.1: source-map@~0.5.1, source-map@~0.5.3: version "0.5.7" resolved "https://registry.yarnpkg.com/source-map/-/source-map-0.5.7.tgz#8a039d2d1021d22d1ea14c80d8ea468ba2ef3fcc" - integrity sha1-igOdLRAh0i0eoUyA2OpGi6LvP8w= + integrity sha512-LbrmJOMUSdEVxIKvdcJzQC+nQhe8FUZQTXQy6+I75skNgn3OoQ0DZA8YnFa7gp8tqtL3KPf1kmo0R5DoApeSGQ== sprintf-js@~1.0.2: version "1.0.3" resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c" - integrity sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw= + integrity sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g== sshpk@^1.7.0: version "1.17.0" @@ -6043,7 +6003,7 @@ stream-browserify@^3.0.0: stream-combiner2@^1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/stream-combiner2/-/stream-combiner2-1.1.1.tgz#fb4d8a1420ea362764e21ad4780397bebcb41cbe" - integrity sha1-+02KFCDqNidk4hrUeAOXvry0HL4= + integrity sha512-3PnJbYgS56AeWgtKF5jtJRT6uFJe56Z0Hc5Ngg/6sI6rIt8iiMBTa9cvdyFfpMQjaVHr8dusbNeFGIIonxOvKw== dependencies: duplexer2 "~0.1.0" readable-stream "^2.0.2" @@ -6125,14 +6085,14 @@ strip-ansi@^6.0.0, strip-ansi@^6.0.1: strip-bom@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" - integrity sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4= + integrity sha512-kwrX1y7czp1E69n2ajbG65mIo9dqvJ+8aBQXOGVxqwvNbsXdFM6Lq37dLAY3mknUwru8CfcCbfOLL/gMo+fi3g== dependencies: is-utf8 "^0.2.0" strip-bom@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-3.0.0.tgz#2334c18e9c759f7bdd56fdef7e9ae3d588e68ed3" - integrity sha1-IzTBjpx1n3vdVv3vfprj1YjmjtM= + integrity sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA== strip-bom@^4.0.0: version "4.0.0" @@ -6154,7 +6114,7 @@ strip-indent@^3.0.0: strip-json-comments@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/strip-json-comments/-/strip-json-comments-2.0.1.tgz#3c531942e908c2697c0ec344858c286c7ca0a60a" - integrity sha1-PFMZQukIwml8DsNEhYwobHygpgo= + integrity sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ== strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: version "3.1.1" @@ -6164,7 +6124,7 @@ strip-json-comments@^3.1.0, strip-json-comments@^3.1.1: subarg@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/subarg/-/subarg-1.0.0.tgz#f62cf17581e996b48fc965699f54c06ae268b8d2" - integrity sha1-9izxdYHplrSPyWVpn1TAauJouNI= + integrity sha512-RIrIdRY0X1xojthNcVtgT9sjpOGagEUKpZdgBUi054OEPFo282yg+zE+t1Rj3+RqKq2xStL7uUHhY+AjbC4BXg== dependencies: minimist "^1.1.0" @@ -6212,7 +6172,7 @@ syntax-error@^1.1.1: taffydb@2.6.2: version "2.6.2" resolved "https://registry.yarnpkg.com/taffydb/-/taffydb-2.6.2.tgz#7cbcb64b5a141b6a2efc2c5d2c67b4e150b2a268" - integrity sha1-fLy2S1oUG2ou/CxdLGe04VCyomg= + integrity sha512-y3JaeRSplks6NYQuCOj3ZFMO3j60rTwbuKCvZxsAraGYH2epusatvZ0baZYA01WsGqJBq/Dl6vOrMUJqyMj8kA== terminal-link@^2.0.0: version "2.1.1" @@ -6223,9 +6183,9 @@ terminal-link@^2.0.0: supports-hyperlinks "^2.0.0" terser@^5.5.1: - version "5.14.0" - resolved "https://registry.yarnpkg.com/terser/-/terser-5.14.0.tgz#eefeec9af5153f55798180ee2617f390bdd285e2" - integrity sha512-JC6qfIEkPBd9j1SMO3Pfn+A6w2kQV54tv+ABQLgZr7dA3k/DL/OBoYSWxzVpZev3J+bUHXfr55L8Mox7AaNo6g== + version "5.14.1" + resolved "https://registry.yarnpkg.com/terser/-/terser-5.14.1.tgz#7c95eec36436cb11cf1902cc79ac564741d19eca" + integrity sha512-+ahUAE+iheqBTDxXhTisdA8hgvbEG1hHOQ9xmNjeUJSoi6DU/gMrKNcfZjHkyY6Alnuyc+ikYJaxxfHkT3+WuQ== dependencies: "@jridgewell/source-map" "^0.3.2" acorn "^8.5.0" @@ -6244,7 +6204,7 @@ test-exclude@^6.0.0: text-table@^0.2.0: version "0.2.0" resolved "https://registry.yarnpkg.com/text-table/-/text-table-0.2.0.tgz#7f5ee823ae805207c00af2df4a84ec3fcfa570b4" - integrity sha1-f17oI66AUgfACvLfSoTsP8+lcLQ= + integrity sha512-N+8UisAXDGk8PFXP4HAzVR9nbfmVJ3zYLAWiTIoqC5v5isinhr+r5uaO8+7r3BMfuNIufIsA7RdpVgacC2cSpw== throat@^6.0.1: version "6.0.1" @@ -6262,17 +6222,17 @@ through2@^2.0.0: "through@>=2.2.7 <3": version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5" - integrity sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU= + integrity sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg== through@~2.2.7: version "2.2.7" resolved "https://registry.yarnpkg.com/through/-/through-2.2.7.tgz#6e8e21200191d4eb6a99f6f010df46aa1c6eb2bd" - integrity sha1-bo4hIAGR1OtqmfbwEN9Gqhxusr0= + integrity sha512-JIR0m0ybkmTcR8URann+HbwKmodP+OE8UCbsifQDYMLD5J3em1Cdn3MYPpbEd5elGDwmP98T+WbqP/tvzA5Mjg== timers-browserify@^1.0.1: version "1.4.2" resolved "https://registry.yarnpkg.com/timers-browserify/-/timers-browserify-1.4.2.tgz#c9c58b575be8407375cb5e2462dacee74359f41d" - integrity sha1-ycWLV1voQHN1y14kYtrO50NZ9B0= + integrity sha512-PIxwAupJZiYU4JmVZYwXp9FKsHMXb5h0ZEFyuXTAn8WLHOlcij+FEcbrvDsom1o5dr1YggEtFbECvGCW2sT53Q== dependencies: process "~0.11.0" @@ -6292,12 +6252,12 @@ tmpl@1.0.5: to-fast-properties@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-1.0.3.tgz#b83571fa4d8c25b82e231b06e3a3055de4ca1a47" - integrity sha1-uDVx+k2MJbguIxsG46MFXeTKGkc= + integrity sha512-lxrWP8ejsq+7E3nNjwYmUBMAgjMTZoTI+sdBOpvNyijeDLa29LUn9QaoXAHv4+Z578hbmHHJKZknzxVtvo77og== to-fast-properties@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/to-fast-properties/-/to-fast-properties-2.0.0.tgz#dc5e698cbd079265bc73e0377681a4e4e83f616e" - integrity sha1-3F5pjL0HkmW8c+A3doGk5Og/YW4= + integrity sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog== to-regex-range@^5.0.1: version "5.0.1" @@ -6309,7 +6269,7 @@ to-regex-range@^5.0.1: token-stream@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/token-stream/-/token-stream-0.0.1.tgz#ceeefc717a76c4316f126d0b9dbaa55d7e7df01a" - integrity sha1-zu78cXp2xDFvEm0LnbqlXX598Bo= + integrity sha512-nfjOAu/zAWmX9tgwi5NRp7O7zTDUD1miHiB40klUnAh9qnL1iXdgzcz/i5dMaL5jahcBAaSfmNOBBJBLJW8TEg== tough-cookie@~2.5.0: version "2.5.0" @@ -6329,7 +6289,7 @@ tr46@^2.1.0: tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" - integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o= + integrity sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw== ts-map@^1.0.3: version "1.0.3" @@ -6349,7 +6309,7 @@ tsconfig-paths@^3.14.1: tsconfig@^5.0.3: version "5.0.3" resolved "https://registry.yarnpkg.com/tsconfig/-/tsconfig-5.0.3.tgz#5f4278e701800967a8fc383fd19648878f2a6e3a" - integrity sha1-X0J45wGACWeo/Dg/0ZZIh48qbjo= + integrity sha512-Cq65A3kVp6BbsUgg9DRHafaGmbMb9EhAc7fjWvudNWKjkbWrt43FnrtZt6awshH1R0ocfF2Z0uxock3lVqEgOg== dependencies: any-promise "^1.3.0" parse-json "^2.2.0" @@ -6393,7 +6353,7 @@ tty-browserify@0.0.1: tunnel-agent@^0.6.0: version "0.6.0" resolved "https://registry.yarnpkg.com/tunnel-agent/-/tunnel-agent-0.6.0.tgz#27a5dea06b36b04a0a9966774b290868f0fc40fd" - integrity sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0= + integrity sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w== dependencies: safe-buffer "^5.0.1" @@ -6405,7 +6365,7 @@ tunnel@^0.0.6: tweetnacl@^0.14.3, tweetnacl@~0.14.0: version "0.14.5" resolved "https://registry.yarnpkg.com/tweetnacl/-/tweetnacl-0.14.5.tgz#5ae68177f192d4456269d108afa93ff8743f4f64" - integrity sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q= + integrity sha512-KXXFFdAbFXY4geFIwoyNK+f5Z1b7swfXABfL7HXCmoIWMKU3dmS26672A4EeQtDzLKy7SXmfBu51JolvEKwtGA== type-check@^0.4.0, type-check@~0.4.0: version "0.4.0" @@ -6442,7 +6402,7 @@ type@^2.5.0: typedarray@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777" - integrity sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c= + integrity sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA== typescript@^3.2.2: version "3.9.10" @@ -6450,9 +6410,9 @@ typescript@^3.2.2: integrity sha512-w6fIxVE/H1PkLKcCPsFqKE7Kv7QUwhU8qQY2MueZXWx5cPZdwFupLgKK3vntcK98BtNHZtAF4LA/yl2a7k8R6Q== typescript@^4.5.3, typescript@^4.5.4: - version "4.7.2" - resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.2.tgz#1f9aa2ceb9af87cca227813b4310fff0b51593c4" - integrity sha512-Mamb1iX2FDUpcTRzltPxgWMKy3fhg0TN378ylbktPGPK/99KbDtMQ4W1hwgsbPAsG3a0xKa1vmw4VKZQbkvz5A== + version "4.7.4" + resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.7.4.tgz#1a88596d1cf47d59507a1bcdfb5b9dfe4d488235" + integrity sha512-C0WQT0gezHuw6AdY1M2jxUO83Rjf0HP7Sk1DtXj6j1EwkQNZrHAg2XPWlq62oqEhYvONq5pkC2Y9oPljWToLmQ== typeson-registry@^1.0.0-alpha.20: version "1.0.0-alpha.39" @@ -6476,7 +6436,7 @@ uc.micro@^1.0.1, uc.micro@^1.0.5: uglify-js@^2.6.1: version "2.8.29" resolved "https://registry.yarnpkg.com/uglify-js/-/uglify-js-2.8.29.tgz#29c5733148057bb4e1f75df35b7a9cb72e6a59dd" - integrity sha1-KcVzMUgFe7Th913zW3qcty5qWd0= + integrity sha512-qLq/4y2pjcU3vhlhseXGGJ7VbFO4pBANu0kwl8VCa9KEI0V8VfZIx2Fy3w01iSTA/pGwKZSmu/+I4etLNDdt5w== dependencies: source-map "~0.5.1" yargs "~3.10.0" @@ -6486,7 +6446,7 @@ uglify-js@^2.6.1: uglify-to-browserify@~1.0.0: version "1.0.2" resolved "https://registry.yarnpkg.com/uglify-to-browserify/-/uglify-to-browserify-1.0.2.tgz#6e0924d6bda6b5afe349e39a6d632850a0f882b7" - integrity sha1-bgkk1r2mta/jSeOabWMoUKD4grc= + integrity sha512-vb2s1lYx2xBtUgy+ta+b2J/GLVUR+wmpINwHePmPRhOsIVCG2wDzKJ0n14GslH1BifsqVzSOwQhRaCAsZ/nI4Q== umd@^3.0.0: version "3.0.3" @@ -6515,9 +6475,9 @@ undeclared-identifiers@^1.1.2: xtend "^4.0.1" underscore@^1.13.2, underscore@~1.13.2: - version "1.13.3" - resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.3.tgz#54bc95f7648c5557897e5e968d0f76bc062c34ee" - integrity sha512-QvjkYpiD+dJJraRA8+dGAU4i7aBbb2s0S3jA45TFOvg2VgqvdCDd/3N6CqA8gluk1W91GLoXg5enMUx560QzuA== + version "1.13.4" + resolved "https://registry.yarnpkg.com/underscore/-/underscore-1.13.4.tgz#7886b46bbdf07f768e0052f1828e1dcab40c0dee" + integrity sha512-BQFnUDuAQ4Yf/cYY5LNrK9NCJFKriaRbD9uR1fTeXnBeoa97W0i41qkZfGO9pSo8I5KzjAcSY2XYtdf0oKd7KQ== unhomoglyph@^1.0.6: version "1.0.6" @@ -6552,6 +6512,14 @@ universal-user-agent@^6.0.0: resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== +update-browserslist-db@^1.0.4: + version "1.0.4" + resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.4.tgz#dbfc5a789caa26b1db8990796c2c8ebbce304824" + integrity sha512-jnmO2BEGUjsMOe/Fg9u0oczOe/ppIDZPebzccl1yDWGLFP16Pa1/RM5wEoKYPG2zstNcDuAStejyxsOuKINdGA== + dependencies: + escalade "^3.1.1" + picocolors "^1.0.0" + uri-js@^4.2.2: version "4.4.1" resolved "https://registry.yarnpkg.com/uri-js/-/uri-js-4.4.1.tgz#9b1a52595225859e55f669d928f88c6c57f2a77e" @@ -6562,7 +6530,7 @@ uri-js@^4.2.2: url@~0.11.0: version "0.11.0" resolved "https://registry.yarnpkg.com/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" - integrity sha1-ODjpfPxgUh63PFJajlW/3Z4uKPE= + integrity sha512-kbailJa29QrtXnxgq+DdCEGlbTeYM2eJUxsz6vjZavrCYPMIFHMKQmSKYAIuUK2i7hgPm28a8piX5NTUtM/LKQ== dependencies: punycode "1.3.2" querystring "0.2.0" @@ -6570,12 +6538,12 @@ url@~0.11.0: util-deprecate@^1.0.1, util-deprecate@~1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf" - integrity sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8= + integrity sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw== util@0.10.3: version "0.10.3" resolved "https://registry.yarnpkg.com/util/-/util-0.10.3.tgz#7afb1afe50805246489e3db7fe0ed379336ac0f9" - integrity sha1-evsa/lCAUkZInj23/g7TeTNqwPk= + integrity sha512-5KiHfsmkqacuKjkRkdV7SsfDJ2EGiPsK92s2MhNSY0craxjTdKTtqKsJaCWp4LW33ZZ0OPUv1WO/TFvNQRiQxQ== dependencies: inherits "2.0.1" @@ -6601,19 +6569,19 @@ v8-compile-cache@^2.0.3: resolved "https://registry.yarnpkg.com/v8-compile-cache/-/v8-compile-cache-2.3.0.tgz#2de19618c66dc247dcfb6f99338035d8245a2cee" integrity sha512-l8lCEmLcLYZh4nbunNZvQCJc5pv7+RCwa8q/LdUx8u7lsWvPDKmpodJAJNwkAhJC//dFY48KuIEmjtd4RViDrA== -v8-to-istanbul@^9.0.0: - version "9.0.0" - resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.0.0.tgz#be0dae58719fc53cb97e5c7ac1d7e6d4f5b19511" - integrity sha512-HcvgY/xaRm7isYmyx+lFKA4uQmfUbN0J4M0nNItvzTvH/iQ9kW5j/t4YSR+Ge323/lrgDAWJoF46tzGQHwBHFw== +v8-to-istanbul@^9.0.0, v8-to-istanbul@^9.0.1: + version "9.0.1" + resolved "https://registry.yarnpkg.com/v8-to-istanbul/-/v8-to-istanbul-9.0.1.tgz#b6f994b0b5d4ef255e17a0d17dc444a9f5132fa4" + integrity sha512-74Y4LqY74kLE6IFyIjPtkSTWzUZmj8tdHT9Ii/26dvQ6K9Dl2NbEfj0XgU2sHCtKgt5VupqhlO/5aWuqS+IY1w== dependencies: - "@jridgewell/trace-mapping" "^0.3.7" + "@jridgewell/trace-mapping" "^0.3.12" "@types/istanbul-lib-coverage" "^2.0.1" convert-source-map "^1.6.0" verror@1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/verror/-/verror-1.10.0.tgz#3a105ca17053af55d6e270c1f8288682e18da400" - integrity sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA= + integrity sha512-ZZKSmDAEFOijERBLkmYfJ+vmk3w+7hOLYDNkRCuRuMJGEmqYNCNLyBBFwWKVMhfwaEF3WOd0Zlw86U/WC/+nYw== dependencies: assert-plus "^1.0.0" core-util-is "1.0.2" @@ -6627,7 +6595,7 @@ vm-browserify@^1.0.0: void-elements@^2.0.1: version "2.0.1" resolved "https://registry.yarnpkg.com/void-elements/-/void-elements-2.0.1.tgz#c066afb582bb1cb4128d60ea92392e94d5e9dbec" - integrity sha1-wGavtYK7HLQSjWDqkjkulNXp2+w= + integrity sha512-qZKX4RnBzH2ugr8Lxa7x+0V6XD9Sb/ouARtiasEQCHB1EVU4NXtmHsDDrx1dO4ne5fc3J6EW05BP1Dl0z0iung== vue-docgen-api@^3.26.0: version "3.26.0" @@ -6646,12 +6614,12 @@ vue-docgen-api@^3.26.0: vue-template-compiler "^2.0.0" vue-template-compiler@^2.0.0: - version "2.6.14" - resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.6.14.tgz#a2f0e7d985670d42c9c9ee0d044fed7690f4f763" - integrity sha512-ODQS1SyMbjKoO1JBJZojSw6FE4qnh9rIpUZn2EUT86FKizx9uH5z6uXiIrm4/Nb/gwxTi/o17ZDEGWAXHvtC7g== + version "2.7.1" + resolved "https://registry.yarnpkg.com/vue-template-compiler/-/vue-template-compiler-2.7.1.tgz#ebbefd9cc9a7b9ee9ee6862fa1ec3a9bf3959963" + integrity sha512-ku/H1k1yHAgY0BEdoXVj7xZIjuFSwB2IV3nQWnmUMJ6U1jzK56LPHLWzEe5bTzt0WR0b/rJRkuiig44SUoaBoQ== dependencies: de-indent "^1.0.2" - he "^1.1.0" + he "^1.2.0" vue2-ace-editor@^0.0.15: version "0.0.15" @@ -6660,7 +6628,7 @@ vue2-ace-editor@^0.0.15: dependencies: brace "^0.11.0" -walker@^1.0.7: +walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" integrity sha512-ts/8E8l5b7kY0vlWLewOkDXMmPdLcVV4GmOQLyxuSswIJsweeFZtAsMF7k1Nszz+TYBQrlYRmzOnr398y1JemQ== @@ -6670,7 +6638,7 @@ walker@^1.0.7: webidl-conversions@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871" - integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE= + integrity sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ== webidl-conversions@^4.0.2: version "4.0.2" @@ -6685,7 +6653,7 @@ webidl-conversions@^6.1.0: whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" - integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0= + integrity sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw== dependencies: tr46 "~0.0.3" webidl-conversions "^3.0.0" @@ -6732,12 +6700,12 @@ which@^2.0.1: window-size@0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/window-size/-/window-size-0.1.0.tgz#5438cd2ea93b202efa3a19fe8887aee7c94f9c9d" - integrity sha1-VDjNLqk7IC76Ohn+iIeu58lPnJ0= + integrity sha512-1pTPQDKTdd61ozlKGNCjhNRd+KPmgLSGa3mZTHoOliaGcESD8G1PXhh7c1fgiPjVbNVfgy2Faw4BI8/m0cC8Mg== with@^5.0.0: version "5.1.1" resolved "https://registry.yarnpkg.com/with/-/with-5.1.1.tgz#fa4daa92daf32c4ea94ed453c81f04686b575dfe" - integrity sha1-+k2qktrzLE6pTtRTyB8EaGtXXf4= + integrity sha512-uAnSsFGfSpF6DNhBXStvlZILfHJfJu4eUkfbRGk94kGO1Ta7bg6FwfvoOhhyHAJuFbCw+0xk4uJ3u57jLvlCJg== dependencies: acorn "^3.1.0" acorn-globals "^3.0.0" @@ -6750,7 +6718,7 @@ word-wrap@^1.2.3: wordwrap@0.0.2: version "0.0.2" resolved "https://registry.yarnpkg.com/wordwrap/-/wordwrap-0.0.2.tgz#b79669bb42ecb409f83d583cad52ca17eaa1643f" - integrity sha1-t5Zpu0LstAn4PVg8rVLKF+qhZD8= + integrity sha512-xSBsCeh+g+dinoBv3GAOWM4LcVVO68wLXRanibtBSdUvkGWQRGeE9P7IwU9EmDDi4jA6L44lz15CGMwdw9N5+Q== wrap-ansi@^7.0.0: version "7.0.0" @@ -6764,7 +6732,7 @@ wrap-ansi@^7.0.0: wrappy@1: version "1.0.2" resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" - integrity sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8= + integrity sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ== write-file-atomic@^4.0.1: version "4.0.1" @@ -6777,7 +6745,7 @@ write-file-atomic@^4.0.1: xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" - integrity sha1-eLpyAgApxbyHuKgaPPzXS0ovweU= + integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw== xmlcreate@^2.0.4: version "2.0.4" @@ -6797,7 +6765,7 @@ y18n@^5.0.5: yallist@^2.1.2: version "2.1.2" resolved "https://registry.yarnpkg.com/yallist/-/yallist-2.1.2.tgz#1c11f9218f076089a47dd512f93c6699a6a81d52" - integrity sha1-HBH5IY8HYImkfdUS+TxmmaaoHVI= + integrity sha512-ncTzHV7NvsQZkYe1DW7cbDLm0YpzHmZF5r/iyP3ZnQtMiJ+pjzisCiMNI+Sj+xQF5pXhSHxSB3uDbsBTzY/c2A== yallist@^4.0.0: version "4.0.0" @@ -6843,7 +6811,7 @@ yargs@^17.0.1, yargs@^17.3.1: yargs@~3.10.0: version "3.10.0" resolved "https://registry.yarnpkg.com/yargs/-/yargs-3.10.0.tgz#f7ee7bd857dd7c1d2d38c0e74efbd681d1431fd1" - integrity sha1-9+572FfdfB0tOMDnTvvWgdFDH9E= + integrity sha512-QFzUah88GAGy9lyDKGBqZdkYApt63rCXYBGYnEP4xDJPXNqXXnBDACnbrXnViV6jRSqAePwrATi2i8mfYm4L1A== dependencies: camelcase "^1.0.2" cliui "^2.1.0" From e876482e62421bb6a1ba58078435c6b3d1210f15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 25 Jul 2022 15:51:06 +0200 Subject: [PATCH 215/291] Add local volume control (#2525) --- src/webrtc/callFeed.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 065e392517e..c9133895212 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -44,6 +44,7 @@ export interface ICallFeedOpts { export enum CallFeedEvent { NewStream = "new_stream", MuteStateChanged = "mute_state_changed", + LocalVolumeChanged = "local_volume_changed", VolumeChanged = "volume_changed", Speaking = "speaking", } @@ -51,6 +52,7 @@ export enum CallFeedEvent { type EventHandlerMap = { [CallFeedEvent.NewStream]: (stream: MediaStream) => void; [CallFeedEvent.MuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; + [CallFeedEvent.LocalVolumeChanged]: (localVolume: number) => void; [CallFeedEvent.VolumeChanged]: (volume: number) => void; [CallFeedEvent.Speaking]: (speaking: boolean) => void; }; @@ -66,6 +68,7 @@ export class CallFeed extends TypedEventEmitter private roomId: string; private audioMuted: boolean; private videoMuted: boolean; + private localVolume = 1; private measuringVolumeActivity = false; private audioContext: AudioContext; private analyser: AnalyserNode; @@ -291,4 +294,13 @@ export class CallFeed extends TypedEventEmitter releaseContext(); } } + + public getLocalVolume(): number { + return this.localVolume; + } + + public setLocalVolume(localVolume: number): void { + this.localVolume = localVolume; + this.emit(CallFeedEvent.LocalVolumeChanged, localVolume); + } } From 38a6949e5d7c73f7968e45102c2ea206dff7b85e Mon Sep 17 00:00:00 2001 From: Timo <16718859+toger5@users.noreply.github.com> Date: Mon, 25 Jul 2022 18:44:37 +0200 Subject: [PATCH 216/291] add missing events from reemitter to GroupCall (#2527) --- src/webrtc/groupCall.ts | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index a494447a0fc..1b988174775 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -1,7 +1,14 @@ import { TypedEventEmitter } from "../models/typed-event-emitter"; import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; import { MatrixClient } from "../client"; -import { CallErrorCode, CallEvent, CallState, genCallID, MatrixCall, setTracksEnabled } from "./call"; +import { CallErrorCode, + CallEvent, + CallEventHandlerMap, + CallState, + genCallID, + MatrixCall, + setTracksEnabled, +} from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; @@ -146,7 +153,10 @@ function getCallUserId(call: MatrixCall): string | null { return call.getOpponentMember()?.userId || call.invitee || null; } -export class GroupCall extends TypedEventEmitter { +export class GroupCall extends TypedEventEmitter< + GroupCallEvent | CallEvent, + GroupCallEventHandlerMap & CallEventHandlerMap +> { // Config public activeSpeakerInterval = 1000; public retryCallInterval = 5000; From 9e2e144530975ef163ac0643e078ab75b91ce320 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 28 Jul 2022 09:38:57 +0100 Subject: [PATCH 217/291] Make SDP munging media type specific (#2526) * Make SDP munging media type specific We were trying to apply modifications to all media types which led to confusing warning messages saying opus wasn't present (when it was for the video stream). Make the modifications media-type specific to avoid this. * Make codec * mediatype into enums --- src/webrtc/call.ts | 51 +++++++++++++++++++++++++++++----------------- 1 file changed, 32 insertions(+), 19 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ad97246c8ba..a8ca5baa69c 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -93,14 +93,24 @@ interface AssertedIdentity { displayName: string; } +enum MediaType { + AUDIO = "audio", + VIDEO = "video", +} + +enum CodecName { + OPUS = "opus", + // add more as needed +} + // Used internally to specify modifications to codec parameters in SDP -interface CodecParams { +interface CodecParamsMod { + mediaType: MediaType; + codec: CodecName; enableDtx?: boolean; // true to enable discontinuous transmission, false to disable, undefined to leave as-is maxAverageBitrate?: number; // sets the max average bitrate, or undefined to leave as-is } -type CodecParamMods = Record; - export enum CallState { Fledgling = 'fledgling', InviteSent = 'invite_sent', @@ -269,14 +279,15 @@ export function genCallID(): string { return Date.now().toString() + randomString(16); } -function getCodecParamMods(isPtt: boolean): CodecParamMods { - const mods = { - 'opus': { +function getCodecParamMods(isPtt: boolean): CodecParamsMod[] { + const mods = [ + { + mediaType: "audio", + codec: "opus", enableDtx: true, + maxAverageBitrate: isPtt ? 12000 : undefined, }, - } as CodecParamMods; - - if (isPtt) mods.opus.maxAverageBitrate = 12000; + ] as CodecParamsMod[]; return mods; } @@ -1476,7 +1487,7 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 28 Jul 2022 18:12:11 +0200 Subject: [PATCH 218/291] Add support for audio sharing (#2530) --- src/webrtc/call.ts | 11 ++++++----- src/webrtc/groupCall.ts | 8 ++++---- src/webrtc/mediaHandler.ts | 22 ++++++++++++++-------- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index a8ca5baa69c..526eaf76744 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -51,6 +51,7 @@ import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; import { GroupCallUnknownDeviceError } from './groupCall'; +import { IScreensharingOpts } from "./mediaHandler"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -1111,7 +1112,7 @@ export class MatrixCall extends TypedEventEmitter { + public async setScreensharingEnabled(enabled: boolean, opts?: IScreensharingOpts): Promise { // Skip if there is nothing to do if (enabled && this.isScreensharing()) { logger.warn(`Call ${this.callId} There is already a screensharing stream - there is nothing to do!`); @@ -1123,13 +1124,13 @@ export class MatrixCall extends TypedEventEmitter { logger.debug(`Call ${this.callId} Set screensharing enabled? ${enabled} using replaceTrack()`); if (enabled) { try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + const stream = await this.client.getMediaHandler().getScreensharingStream(opts); if (!stream) return false; const track = stream.getTracks().find((track) => { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 1b988174775..7c3a891a6a4 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -20,6 +20,7 @@ import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; import { CallEventHandlerEvent } from "./callEventHandler"; import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; +import { IScreensharingOpts } from "./mediaHandler"; export enum GroupCallIntent { Ring = "m.ring", @@ -537,7 +538,7 @@ export class GroupCall extends TypedEventEmitter< } public async setScreensharingEnabled( - enabled: boolean, desktopCapturerSourceId?: string, + enabled: boolean, opts: IScreensharingOpts = {}, ): Promise { if (enabled === this.isScreensharing()) { return enabled; @@ -546,8 +547,7 @@ export class GroupCall extends TypedEventEmitter< if (enabled) { try { logger.log("Asking for screensharing permissions..."); - - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + const stream = await this.client.getMediaHandler().getScreensharingStream(opts); for (const track of stream.getTracks()) { const onTrackEnded = () => { @@ -560,7 +560,7 @@ export class GroupCall extends TypedEventEmitter< logger.log("Screensharing permissions granted. Setting screensharing enabled on all calls"); - this.localDesktopCapturerSourceId = desktopCapturerSourceId; + this.localDesktopCapturerSourceId = opts.desktopCapturerSourceId; this.localScreenshareFeed = new CallFeed({ client: this.client, roomId: this.room.roomId, diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 61a8e106da2..1b55cfbb908 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -30,6 +30,11 @@ export type MediaHandlerEventHandlerMap = { [MediaHandlerEvent.LocalStreamsChanged]: () => void; }; +export interface IScreensharingOpts { + desktopCapturerSourceId?: string; + audio?: boolean; +} + export class MediaHandler extends TypedEventEmitter< MediaHandlerEvent.LocalStreamsChanged, MediaHandlerEventHandlerMap > { @@ -254,20 +259,20 @@ export class MediaHandler extends TypedEventEmitter< * @param reusable is allowed to be reused by the MediaHandler * @returns {MediaStream} based on passed parameters */ - public async getScreensharingStream(desktopCapturerSourceId: string, reusable = true): Promise { + public async getScreensharingStream(opts: IScreensharingOpts = {}, reusable = true): Promise { let stream: MediaStream; if (this.screensharingStreams.length === 0) { - const screenshareConstraints = this.getScreenshareContraints(desktopCapturerSourceId); + const screenshareConstraints = this.getScreenshareContraints(opts); if (!screenshareConstraints) return null; - if (desktopCapturerSourceId) { + if (opts.desktopCapturerSourceId) { // We are using Electron - logger.debug("Getting screensharing stream using getUserMedia()", desktopCapturerSourceId); + logger.debug("Getting screensharing stream using getUserMedia()", opts); stream = await navigator.mediaDevices.getUserMedia(screenshareConstraints); } else { // We are not using Electron - logger.debug("Getting screensharing stream using getDisplayMedia()"); + logger.debug("Getting screensharing stream using getDisplayMedia()", opts); stream = await navigator.mediaDevices.getDisplayMedia(screenshareConstraints); } } else { @@ -352,11 +357,12 @@ export class MediaHandler extends TypedEventEmitter< }; } - private getScreenshareContraints(desktopCapturerSourceId?: string): DesktopCapturerConstraints { + private getScreenshareContraints(opts: IScreensharingOpts): DesktopCapturerConstraints { + const { desktopCapturerSourceId, audio } = opts; if (desktopCapturerSourceId) { logger.debug("Using desktop capturer source", desktopCapturerSourceId); return { - audio: false, + audio, video: { mandatory: { chromeMediaSource: "desktop", @@ -367,7 +373,7 @@ export class MediaHandler extends TypedEventEmitter< } else { logger.debug("Not using desktop capturer source"); return { - audio: false, + audio, video: true, }; } From 22c5999fed9d0e8b8f31f510c5048e04cde40e80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 1 Aug 2022 18:45:14 +0200 Subject: [PATCH 219/291] Delint group calls (#2554) --- spec/unit/crypto/algorithms/megolm.spec.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/spec/unit/crypto/algorithms/megolm.spec.ts b/spec/unit/crypto/algorithms/megolm.spec.ts index 3e2acca379d..8e8250c445a 100644 --- a/spec/unit/crypto/algorithms/megolm.spec.ts +++ b/spec/unit/crypto/algorithms/megolm.spec.ts @@ -360,8 +360,11 @@ describe("MegolmDecryption", function() { // Splice the real method onto the mock object as megolm uses this method // on the crypto class in order to encrypt / start sessions + // @ts-ignore Mock mockCrypto.encryptAndSendToDevices = Crypto.prototype.encryptAndSendToDevices; + // @ts-ignore Mock mockCrypto.olmDevice = olmDevice; + // @ts-ignore Mock mockCrypto.baseApis = mockBaseApis; mockRoom = { From 2cc51e0db76aad7af378bc92fadd729d06c394c7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 4 Aug 2022 17:28:54 +0100 Subject: [PATCH 220/291] Merge changes from develop (#2563) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Prepare changelog for v19.2.0-rc.1 * v19.2.0-rc.1 * Sliding sync: add missing filters from latest MSC * Gracefully handle missing room_ids * Prepare changelog for v19.2.0 * v19.2.0 * Resetting package fields for development * Use EventType enum values instead of hardcoded strings (#2557) * Retry to-device messages (#2549) * Retry to-device messages This adds a queueToDevice API alongside sendToDevice which is a much higher-level API that adds the messages to a queue, stored in persistent storage, and retries them periodically. Also converts sending of megolm keys to use the new API. Other uses of sendToDevice are nopt converted in this PR, but could be later. Requires https://github.com/matrix-org/matrix-mock-request/pull/17 * Bump matrix-mock-request * Add more waits to make indexeddb tests pass * Switch some test expectations to queueToDevice * Stop straight away if the client has been stopped Hopefully will fix tests being flakey and logging after tests have finished. * Add return types & fix constant usage * Fix return type Co-authored-by: Germain * Fix return type Co-authored-by: Germain * Fix return type Co-authored-by: Germain * Stop the client in all test cases Co-authored-by: Germain * Add support for sending user-defined encrypted to-device messages (#2528) * Add support for sending user-defined encrypted to-device messages This is a port of the same change from the robertlong/group-call branch. * Fix tests * Expose the method in MatrixClient * Fix a code smell * Fix types * Test the MatrixClient method * Fix some types in Crypto test suite * Test the Crypto method * Fix tests * Upgrade matrix-mock-request * Move useRealTimers to afterEach * Remove stream-replacement (#2551) * Reintroduce setNewStream method, fix test, update yarn.lock Co-authored-by: RiotRobot Co-authored-by: Kegan Dougal Co-authored-by: Germain Co-authored-by: Robin Co-authored-by: Šimon Brandner --- CHANGELOG.md | 15 + package.json | 4 +- spec/unit/crypto.spec.ts | 130 +++++++- spec/unit/crypto/algorithms/megolm.spec.ts | 9 +- spec/unit/matrix-client.spec.ts | 16 + spec/unit/queueToDevice.spec.ts | 338 +++++++++++++++++++++ spec/unit/webrtc/call.spec.ts | 125 ++++---- spec/unit/webrtc/callFeed.spec.ts | 78 +++-- src/ToDeviceMessageQueue.ts | 125 ++++++++ src/client.ts | 66 +++- src/crypto/algorithms/megolm.ts | 18 +- src/crypto/index.ts | 137 ++++----- src/models/ToDeviceMessage.ts | 38 +++ src/models/room.ts | 6 +- src/scheduler.ts | 2 +- src/sliding-sync.ts | 5 +- src/store/index.ts | 75 +++-- src/store/indexeddb-backend.ts | 4 + src/store/indexeddb-local-backend.ts | 44 ++- src/store/indexeddb-remote-backend.ts | 13 + src/store/indexeddb-store-worker.ts | 9 + src/store/indexeddb.ts | 13 + src/store/memory.ts | 25 ++ src/store/stub.ts | 13 + src/webrtc/call.ts | 102 +++---- src/webrtc/callFeed.ts | 4 +- yarn.lock | 31 +- 27 files changed, 1155 insertions(+), 290 deletions(-) create mode 100644 spec/unit/queueToDevice.spec.ts create mode 100644 src/ToDeviceMessageQueue.ts create mode 100644 src/models/ToDeviceMessage.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 065109f5194..12ebda0ba30 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,18 @@ +Changes in [19.2.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.2.0) (2022-08-02) +================================================================================================== + +## 🦖 Deprecations + * Remove unstable support for `m.room_key.withheld` ([\#2512](https://github.com/matrix-org/matrix-js-sdk/pull/2512)). Fixes #2233. + +## ✨ Features + * Sliding sync: add missing filters from latest MSC ([\#2555](https://github.com/matrix-org/matrix-js-sdk/pull/2555)). + * Use stable prefixes for MSC3827 ([\#2537](https://github.com/matrix-org/matrix-js-sdk/pull/2537)). + * Add support for MSC3575: Sliding Sync ([\#2242](https://github.com/matrix-org/matrix-js-sdk/pull/2242)). + +## 🐛 Bug Fixes + * Correct the units in TURN servers expiry documentation ([\#2520](https://github.com/matrix-org/matrix-js-sdk/pull/2520)). + * Re-insert room IDs when decrypting bundled redaction events returned by `/sync` ([\#2531](https://github.com/matrix-org/matrix-js-sdk/pull/2531)). Contributed by @duxovni. + Changes in [19.1.0](https://github.com/matrix-org/matrix-js-sdk/releases/tag/v19.1.0) (2022-07-26) ================================================================================================== diff --git a/package.json b/package.json index a8ddbc092d4..787e776bd68 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "matrix-js-sdk", - "version": "19.1.0", + "version": "19.2.0", "description": "Matrix Client-Server SDK for Javascript", "engines": { "node": ">=12.9.0" @@ -104,7 +104,7 @@ "jest-localstorage-mock": "^2.4.6", "jest-sonar-reporter": "^2.0.0", "jsdoc": "^3.6.6", - "matrix-mock-request": "^2.1.0", + "matrix-mock-request": "^2.1.2", "rimraf": "^3.0.2", "terser": "^5.5.1", "tsify": "^5.0.2", diff --git a/spec/unit/crypto.spec.ts b/spec/unit/crypto.spec.ts index b579b7f3807..19217cdda83 100644 --- a/spec/unit/crypto.spec.ts +++ b/spec/unit/crypto.spec.ts @@ -2,6 +2,7 @@ import '../olm-loader'; // eslint-disable-next-line no-restricted-imports import { EventEmitter } from "events"; +import { MatrixClient } from "../../src/client"; import { Crypto } from "../../src/crypto"; import { MemoryCryptoStore } from "../../src/crypto/store/memory-crypto-store"; import { MockStorageApi } from "../MockStorageApi"; @@ -64,6 +65,10 @@ describe("Crypto", function() { return Olm.init(); }); + afterEach(() => { + jest.useRealTimers(); + }); + it("Crypto exposes the correct olm library version", function() { expect(Crypto.getOlmVersion()[0]).toEqual(3); }); @@ -225,8 +230,8 @@ describe("Crypto", function() { }); describe('Key requests', function() { - let aliceClient; - let bobClient; + let aliceClient: MatrixClient; + let bobClient: MatrixClient; beforeEach(async function() { aliceClient = (new TestClient( @@ -313,7 +318,7 @@ describe("Crypto", function() { expect(events[0].getContent().msgtype).toBe("m.bad.encrypted"); expect(events[1].getContent().msgtype).not.toBe("m.bad.encrypted"); - const cryptoStore = bobClient.cryptoStore; + const cryptoStore = bobClient.crypto.cryptoStore; const eventContent = events[0].getWireContent(); const senderKey = eventContent.sender_key; const sessionId = eventContent.session_id; @@ -383,9 +388,9 @@ describe("Crypto", function() { const ksEvent = await keyshareEventForEvent(aliceClient, event, 1); ksEvent.getContent().sender_key = undefined; // test - bobClient.crypto.addInboundGroupSession = jest.fn(); + bobClient.crypto.olmDevice.addInboundGroupSession = jest.fn(); await bobDecryptor.onRoomKeyEvent(ksEvent); - expect(bobClient.crypto.addInboundGroupSession).not.toHaveBeenCalled(); + expect(bobClient.crypto.olmDevice.addInboundGroupSession).not.toHaveBeenCalled(); }); it("creates a new keyshare request if we request a keyshare", async function() { @@ -401,7 +406,7 @@ describe("Crypto", function() { }, }); await aliceClient.cancelAndResendEventRoomKeyRequest(event); - const cryptoStore = aliceClient.cryptoStore; + const cryptoStore = aliceClient.crypto.cryptoStore; const roomKeyRequestBody = { algorithm: olmlib.MEGOLM_ALGORITHM, room_id: "!someroom", @@ -425,7 +430,8 @@ describe("Crypto", function() { }, }); // replace Alice's sendToDevice function with a mock - aliceClient.sendToDevice = jest.fn().mockResolvedValue(undefined); + const aliceSendToDevice = jest.fn().mockResolvedValue(undefined); + aliceClient.sendToDevice = aliceSendToDevice; aliceClient.startClient(); // make a room key request, and record the transaction ID for the @@ -434,11 +440,12 @@ describe("Crypto", function() { // key requests get queued until the sync has finished, but we don't // let the client set up enough for that to happen, so gut-wrench a bit // to force it to send now. + // @ts-ignore aliceClient.crypto.outgoingRoomKeyRequestManager.sendQueuedRequests(); jest.runAllTimers(); await Promise.resolve(); - expect(aliceClient.sendToDevice).toBeCalledTimes(1); - const txnId = aliceClient.sendToDevice.mock.calls[0][2]; + expect(aliceSendToDevice).toBeCalledTimes(1); + const txnId = aliceSendToDevice.mock.calls[0][2]; // give the room key request manager time to update the state // of the request @@ -451,8 +458,8 @@ describe("Crypto", function() { // cancelAndResend will call sendToDevice twice: // the first call to sendToDevice will be the cancellation // the second call to sendToDevice will be the key request - expect(aliceClient.sendToDevice).toBeCalledTimes(3); - expect(aliceClient.sendToDevice.mock.calls[2][2]).not.toBe(txnId); + expect(aliceSendToDevice).toBeCalledTimes(3); + expect(aliceSendToDevice.mock.calls[2][2]).not.toBe(txnId); }); }); @@ -480,4 +487,105 @@ describe("Crypto", function() { client.stopClient(); }); }); + + describe("encryptAndSendToDevices", () => { + let client: TestClient; + let ensureOlmSessionsForDevices: jest.SpiedFunction; + let encryptMessageForDevice: jest.SpiedFunction; + const payload = { hello: "world" }; + let encryptedPayload: object; + + beforeEach(async () => { + ensureOlmSessionsForDevices = jest.spyOn(olmlib, "ensureOlmSessionsForDevices"); + ensureOlmSessionsForDevices.mockResolvedValue({}); + encryptMessageForDevice = jest.spyOn(olmlib, "encryptMessageForDevice"); + encryptMessageForDevice.mockImplementation(async (...[result,,,,,, payload]) => { + result.plaintext = JSON.stringify(payload); + }); + + client = new TestClient("@alice:example.org", "aliceweb"); + await client.client.initCrypto(); + + encryptedPayload = { + algorithm: "m.olm.v1.curve25519-aes-sha2", + sender_key: client.client.crypto.olmDevice.deviceCurve25519Key, + ciphertext: { plaintext: JSON.stringify(payload) }, + }; + }); + + afterEach(async () => { + ensureOlmSessionsForDevices.mockRestore(); + encryptMessageForDevice.mockRestore(); + await client.stop(); + }); + + it("encrypts and sends to devices", async () => { + client.httpBackend + .when("PUT", "/sendToDevice/m.room.encrypted", { + messages: { + "@bob:example.org": { + bobweb: encryptedPayload, + bobmobile: encryptedPayload, + }, + "@carol:example.org": { + caroldesktop: encryptedPayload, + }, + }, + }) + .respond(200, {}); + + await Promise.all([ + client.client.encryptAndSendToDevices( + [ + { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobweb") }, + { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobmobile") }, + { userId: "@carol:example.org", deviceInfo: new DeviceInfo("caroldesktop") }, + ], + payload, + ), + client.httpBackend.flushAllExpected(), + ]); + }); + + it("sends nothing to devices that couldn't be encrypted to", async () => { + encryptMessageForDevice.mockImplementation(async (...[result,,,, userId, device, payload]) => { + // Refuse to encrypt to Carol's desktop device + if (userId === "@carol:example.org" && device.deviceId === "caroldesktop") return; + result.plaintext = JSON.stringify(payload); + }); + + client.httpBackend + .when("PUT", "/sendToDevice/m.room.encrypted", { + // Carol is nowhere to be seen + messages: { "@bob:example.org": { bobweb: encryptedPayload } }, + }) + .respond(200, {}); + + await Promise.all([ + client.client.encryptAndSendToDevices( + [ + { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobweb") }, + { userId: "@carol:example.org", deviceInfo: new DeviceInfo("caroldesktop") }, + ], + payload, + ), + client.httpBackend.flushAllExpected(), + ]); + }); + + it("no-ops if no devices can be encrypted to", async () => { + // Refuse to encrypt to anybody + encryptMessageForDevice.mockResolvedValue(undefined); + + // Get the room keys version request out of the way + client.httpBackend.when("GET", "/room_keys/version").respond(404, {}); + await client.httpBackend.flush("/room_keys/version", 1); + + await client.client.encryptAndSendToDevices( + [{ userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobweb") }], + payload, + ); + client.httpBackend.verifyNoOutstandingRequests(); + }); + }); }); diff --git a/spec/unit/crypto/algorithms/megolm.spec.ts b/spec/unit/crypto/algorithms/megolm.spec.ts index 8e8250c445a..9aa3c5c785f 100644 --- a/spec/unit/crypto/algorithms/megolm.spec.ts +++ b/spec/unit/crypto/algorithms/megolm.spec.ts @@ -59,6 +59,7 @@ describe("MegolmDecryption", function() { mockBaseApis = { claimOneTimeKeys: jest.fn(), sendToDevice: jest.fn(), + queueToDevice: jest.fn(), } as unknown as MockedObject; const cryptoStore = new MemoryCryptoStore(); @@ -179,6 +180,7 @@ describe("MegolmDecryption", function() { }); mockBaseApis.sendToDevice.mockReset(); + mockBaseApis.queueToDevice.mockReset(); // do the share megolmDecryption.shareKeysWithDevice(keyRequest); @@ -324,6 +326,7 @@ describe("MegolmDecryption", function() { }, }); mockBaseApis.sendToDevice.mockResolvedValue(undefined); + mockBaseApis.queueToDevice.mockResolvedValue(undefined); aliceDeviceInfo = { deviceId: 'aliceDevice', @@ -413,7 +416,7 @@ describe("MegolmDecryption", function() { expect(mockCrypto.downloadKeys).toHaveBeenCalledWith( ['@alice:home.server'], false, ); - expect(mockBaseApis.sendToDevice).toHaveBeenCalled(); + expect(mockBaseApis.queueToDevice).toHaveBeenCalled(); expect(mockBaseApis.claimOneTimeKeys).toHaveBeenCalledWith( [['@alice:home.server', 'aliceDevice']], 'signed_curve25519', 2000, ); @@ -456,7 +459,7 @@ describe("MegolmDecryption", function() { 'YWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWFhYWI', ); - mockBaseApis.sendToDevice.mockClear(); + mockBaseApis.queueToDevice.mockClear(); await megolmEncryption.reshareKeyWithDevice( olmDevice.deviceCurve25519Key, ct1.session_id, @@ -464,7 +467,7 @@ describe("MegolmDecryption", function() { aliceDeviceInfo, ); - expect(mockBaseApis.sendToDevice).not.toHaveBeenCalled(); + expect(mockBaseApis.queueToDevice).not.toHaveBeenCalled(); }); }); }); diff --git a/spec/unit/matrix-client.spec.ts b/spec/unit/matrix-client.spec.ts index fbe8c67d7e7..db337750113 100644 --- a/spec/unit/matrix-client.spec.ts +++ b/spec/unit/matrix-client.spec.ts @@ -27,6 +27,7 @@ import { UNSTABLE_MSC3089_TREE_SUBTYPE, } from "../../src/@types/event"; import { MEGOLM_ALGORITHM } from "../../src/crypto/olmlib"; +import { Crypto } from "../../src/crypto"; import { EventStatus, MatrixEvent } from "../../src/models/event"; import { Preset } from "../../src/@types/partials"; import { ReceiptType } from "../../src/@types/read_receipts"; @@ -1297,4 +1298,19 @@ describe("MatrixClient", function() { expect(result!.aliases).toEqual(response.aliases); }); }); + + describe("encryptAndSendToDevices", () => { + it("throws an error if crypto is unavailable", () => { + client.crypto = undefined; + expect(() => client.encryptAndSendToDevices([], {})).toThrow(); + }); + + it("is an alias for the crypto method", async () => { + client.crypto = testUtils.mock(Crypto, "Crypto"); + const deviceInfos = []; + const payload = {}; + await client.encryptAndSendToDevices(deviceInfos, payload); + expect(client.crypto.encryptAndSendToDevices).toHaveBeenLastCalledWith(deviceInfos, payload); + }); + }); }); diff --git a/spec/unit/queueToDevice.spec.ts b/spec/unit/queueToDevice.spec.ts new file mode 100644 index 00000000000..ff22d29d48d --- /dev/null +++ b/spec/unit/queueToDevice.spec.ts @@ -0,0 +1,338 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import MockHttpBackend from 'matrix-mock-request'; +import { indexedDB as fakeIndexedDB } from 'fake-indexeddb'; + +import { IHttpOpts, IndexedDBStore, MatrixEvent, MemoryStore, Room } from "../../src"; +import { MatrixClient } from "../../src/client"; +import { ToDeviceBatch } from '../../src/models/ToDeviceMessage'; +import { logger } from '../../src/logger'; +import { IStore } from '../../src/store'; + +const FAKE_USER = "@alice:example.org"; +const FAKE_DEVICE_ID = "AAAAAAAA"; +const FAKE_PAYLOAD = { + "foo": 42, +}; +const EXPECTED_BODY = { + messages: { + [FAKE_USER]: { + [FAKE_DEVICE_ID]: FAKE_PAYLOAD, + }, + }, +}; + +const FAKE_MSG = { + userId: FAKE_USER, + deviceId: FAKE_DEVICE_ID, + payload: FAKE_PAYLOAD, +}; + +enum StoreType { + Memory = 'Memory', + IndexedDB = 'IndexedDB', +} + +// Jest now uses @sinonjs/fake-timers which exposes tickAsync() and a number of +// other async methods which break the event loop, letting scheduled promise +// callbacks run. Unfortunately, Jest doesn't expose these, so we have to do +// it manually (this is what sinon does under the hood). We do both in a loop +// until the thing we expect happens: hopefully this is the least flakey way +// and avoids assuming anything about the app's behaviour. +const realSetTimeout = setTimeout; +function flushPromises() { + return new Promise(r => { + realSetTimeout(r, 1); + }); +} + +async function flushAndRunTimersUntil(cond: () => boolean) { + while (!cond()) { + await flushPromises(); + if (cond()) break; + jest.advanceTimersToNextTimer(); + } +} + +describe.each([ + [StoreType.Memory], [StoreType.IndexedDB], +])("queueToDevice (%s store)", function(storeType) { + let httpBackend: MockHttpBackend; + let client: MatrixClient; + + beforeEach(async function() { + httpBackend = new MockHttpBackend(); + + let store: IStore; + if (storeType === StoreType.IndexedDB) { + const idbStore = new IndexedDBStore({ indexedDB: fakeIndexedDB }); + await idbStore.startup(); + store = idbStore; + } else { + store = new MemoryStore(); + } + + client = new MatrixClient({ + baseUrl: "https://my.home.server", + accessToken: "my.access.token", + request: httpBackend.requestFn as IHttpOpts["request"], + store, + }); + }); + + afterEach(function() { + jest.useRealTimers(); + client.stopClient(); + }); + + it("sends a to-device message", async function() { + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).check((request) => { + expect(request.data).toEqual(EXPECTED_BODY); + }).respond(200, {}); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + + await httpBackend.flushAllExpected(); + }); + + it("retries on error", async function() { + jest.useFakeTimers(); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(500); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).check((request) => { + expect(request.data).toEqual(EXPECTED_BODY); + }).respond(200, {}); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + await flushAndRunTimersUntil(() => httpBackend.requests.length > 0); + expect(httpBackend.flushSync(null, 1)).toEqual(1); + + await flushAndRunTimersUntil(() => httpBackend.requests.length > 0); + + expect(httpBackend.flushSync(null, 1)).toEqual(1); + }); + + it("stops retrying on 4xx errors", async function() { + jest.useFakeTimers(); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(400); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + await flushAndRunTimersUntil(() => httpBackend.requests.length > 0); + expect(httpBackend.flushSync(null, 1)).toEqual(1); + + // Asserting that another request is never made is obviously + // a bit tricky - we just flush the queue what should hopefully + // be plenty of times and assert that nothing comes through. + let tries = 0; + await flushAndRunTimersUntil(() => ++tries === 10); + + expect(httpBackend.requests.length).toEqual(0); + }); + + it("honours ratelimiting", async function() { + jest.useFakeTimers(); + + // pick something obscure enough it's unlikley to clash with a + // retry delay the algorithm uses anyway + const retryDelay = 279 * 1000; + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(429, { + errcode: "M_LIMIT_EXCEEDED", + retry_after_ms: retryDelay, + }); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(200, {}); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + await flushAndRunTimersUntil(() => httpBackend.requests.length > 0); + expect(httpBackend.flushSync(null, 1)).toEqual(1); + await flushPromises(); + + logger.info("Advancing clock to just before expected retry time..."); + + jest.advanceTimersByTime(retryDelay - 1000); + await flushPromises(); + + expect(httpBackend.requests.length).toEqual(0); + + logger.info("Advancing clock past expected retry time..."); + + jest.advanceTimersByTime(2000); + await flushPromises(); + + expect(httpBackend.flushSync(null, 1)).toEqual(1); + }); + + it("retries on retryImmediately()", async function() { + httpBackend.when("GET", "/_matrix/client/versions").respond(200, { + versions: ["r0.0.1"], + }); + + await Promise.all([client.startClient(), httpBackend.flush(null, 1, 20)]); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(500); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(200, {}); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + expect(await httpBackend.flush(null, 1, 1)).toEqual(1); + await flushPromises(); + + client.retryImmediately(); + + expect(await httpBackend.flush(null, 1, 20)).toEqual(1); + }); + + it("retries on when client is started", async function() { + httpBackend.when("GET", "/_matrix/client/versions").respond(200, { + versions: ["r0.0.1"], + }); + + await Promise.all([client.startClient(), httpBackend.flush("/_matrix/client/versions", 1, 20)]); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(500); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(200, {}); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + expect(await httpBackend.flush(null, 1, 1)).toEqual(1); + await flushPromises(); + + client.stopClient(); + await Promise.all([client.startClient(), httpBackend.flush("/_matrix/client/versions", 1, 20)]); + + expect(await httpBackend.flush(null, 1, 20)).toEqual(1); + }); + + it("retries when a message is retried", async function() { + httpBackend.when("GET", "/_matrix/client/versions").respond(200, { + versions: ["r0.0.1"], + }); + + await Promise.all([client.startClient(), httpBackend.flush(null, 1, 20)]); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(500); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).respond(200, {}); + + await client.queueToDevice({ + eventType: "org.example.foo", + batch: [ + FAKE_MSG, + ], + }); + + expect(await httpBackend.flush(null, 1, 1)).toEqual(1); + await flushPromises(); + + const dummyEvent = new MatrixEvent({ + event_id: "!fake:example.org", + }); + const mockRoom = { + updatePendingEvent: jest.fn(), + } as unknown as Room; + client.resendEvent(dummyEvent, mockRoom); + + expect(await httpBackend.flush(null, 1, 20)).toEqual(1); + }); + + it("splits many messages into multiple HTTP requests", async function() { + const batch: ToDeviceBatch = { + eventType: "org.example.foo", + batch: [], + }; + + for (let i = 0; i <= 20; ++i) { + batch.batch.push({ + userId: `@user${i}:example.org`, + deviceId: FAKE_DEVICE_ID, + payload: FAKE_PAYLOAD, + }); + } + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).check((request) => { + expect(Object.keys(request.data.messages).length).toEqual(20); + }).respond(200, {}); + + httpBackend.when( + "PUT", "/sendToDevice/org.example.foo/", + ).check((request) => { + expect(Object.keys(request.data.messages).length).toEqual(1); + }).respond(200, {}); + + await client.queueToDevice(batch); + await httpBackend.flushAllExpected(); + }); +}); diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 6fc8a5a5d81..9fa33e3ece5 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -415,71 +415,6 @@ describe('Call', function() { }).track.id).toBe("video_track"); }); - describe("should handle stream replacement", () => { - it("with both purpose and id", async () => { - await startVoiceCall(client, call); - - call.updateRemoteSDPStreamMetadata({ - "remote_stream1": { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1"); - - call.updateRemoteSDPStreamMetadata({ - "remote_stream2": { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }); - call.pushRemoteFeed(new MockMediaStream("remote_stream2", [])); - - expect(feed?.stream?.id).toBe("remote_stream2"); - }); - - it("with just purpose", async () => { - await startVoiceCall(client, call); - - call.updateRemoteSDPStreamMetadata({ - "remote_stream1": { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1"); - - call.updateRemoteSDPStreamMetadata({ - "remote_stream2": { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }); - call.pushRemoteFeed(new MockMediaStream("remote_stream2", [])); - - expect(feed?.stream?.id).toBe("remote_stream2"); - }); - - it("should not replace purpose is different", async () => { - await startVoiceCall(client, call); - - call.updateRemoteSDPStreamMetadata({ - "remote_stream1": { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); - const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream1"); - - call.updateRemoteSDPStreamMetadata({ - "remote_stream2": { - purpose: SDPStreamMetadataPurpose.Screenshare, - }, - }); - call.pushRemoteFeed(new MockMediaStream("remote_stream2", [])); - - expect(feed?.stream?.id).toBe("remote_stream1"); - }); - }); - it("should handle SDPStreamMetadata changes", async () => { await startVoiceCall(client, call); @@ -794,4 +729,64 @@ describe('Call', function() { expect(supportsMatrixCall()).toBe(false); }); }); + + describe("ignoring streams with ids for which we already have a feed", () => { + const STREAM_ID = "stream_id"; + const FEEDS_CHANGED_CALLBACK = jest.fn(); + + beforeEach(async () => { + await startVoiceCall(client, call); + call.on(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); + jest.spyOn(call, "pushLocalFeed"); + }); + + afterEach(() => { + FEEDS_CHANGED_CALLBACK.mockReset(); + }); + + it("should ignore stream passed to pushRemoteFeed()", async () => { + await call.onAnswerReceived({ + getContent: () => { + return { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, + }, + [SDPStreamMetadataKey]: { + [STREAM_ID]: { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }, + }; + }, + getSender: () => "@test:foo", + }); + + call.pushRemoteFeed(new MockMediaStream(STREAM_ID)); + call.pushRemoteFeed(new MockMediaStream(STREAM_ID)); + + expect(call.getRemoteFeeds().length).toBe(1); + expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); + }); + + it("should ignore stream passed to pushRemoteFeedWithoutMetadata()", async () => { + call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + + expect(call.getRemoteFeeds().length).toBe(1); + expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); + }); + + it("should ignore stream passed to pushNewLocalFeed()", async () => { + call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + + // We already have one local feed from placeVoiceCall() + expect(call.getLocalFeeds().length).toBe(2); + expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); + expect(call.pushLocalFeed).toHaveBeenCalled(); + }); + }); }); diff --git a/spec/unit/webrtc/callFeed.spec.ts b/spec/unit/webrtc/callFeed.spec.ts index e8881781dd9..635fa14fd8f 100644 --- a/spec/unit/webrtc/callFeed.spec.ts +++ b/spec/unit/webrtc/callFeed.spec.ts @@ -15,13 +15,11 @@ limitations under the License. */ import { SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; -import { CallFeed, CallFeedEvent } from "../../../src/webrtc/callFeed"; -import { MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; +import { CallFeed } from "../../../src/webrtc/callFeed"; import { TestClient } from "../../TestClient"; +import { MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; describe("CallFeed", () => { - const roomId = "room_id"; - let client; beforeEach(() => { @@ -32,30 +30,60 @@ describe("CallFeed", () => { client.stop(); }); - it("should handle stream replacement", () => { - const feedNewStreamCallback = jest.fn(); - const feed = new CallFeed({ - client, - roomId, - userId: "user1", - // @ts-ignore Mock - stream: new MockMediaStream("stream1"), - id: "id", - purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: false, - videoMuted: false, + describe("muting", () => { + let feed: CallFeed; + + beforeEach(() => { + feed = new CallFeed({ + client, + roomId: "room1", + userId: "user1", + // @ts-ignore Mock + stream: new MockMediaStream("stream1"), + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: false, + }); + }); + + describe("muting by default", () => { + it("should mute audio by default", () => { + expect(feed.isAudioMuted()).toBeTruthy(); + }); + + it("should mute video by default", () => { + expect(feed.isVideoMuted()).toBeTruthy(); + }); }); - feed.on(CallFeedEvent.NewStream, feedNewStreamCallback); - const replacementStream = new MockMediaStream("stream2"); - // @ts-ignore Mock - feed.setNewStream(replacementStream); - expect(feedNewStreamCallback).toHaveBeenCalledWith(replacementStream); - expect(feed.stream).toBe(replacementStream); + describe("muting after adding a track", () => { + it("should un-mute audio", () => { + // @ts-ignore Mock + feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true)); + expect(feed.isAudioMuted()).toBeFalsy(); + }); - feedNewStreamCallback.mockReset(); + it("should un-mute video", () => { + // @ts-ignore Mock + feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true)); + expect(feed.isVideoMuted()).toBeFalsy(); + }); + }); - replacementStream.addTrack(new MockMediaStreamTrack("track_id", "audio")); - expect(feedNewStreamCallback).toHaveBeenCalledWith(replacementStream); + describe("muting after calling setAudioVideoMuted()", () => { + it("should mute audio by default ", () => { + // @ts-ignore Mock + feed.stream.addTrack(new MockMediaStreamTrack("track", "audio", true)); + feed.setAudioVideoMuted(true, false); + expect(feed.isAudioMuted()).toBeTruthy(); + }); + + it("should mute video by default", () => { + // @ts-ignore Mock + feed.stream.addTrack(new MockMediaStreamTrack("track", "video", true)); + feed.setAudioVideoMuted(false, true); + expect(feed.isVideoMuted()).toBeTruthy(); + }); + }); }); }); diff --git a/src/ToDeviceMessageQueue.ts b/src/ToDeviceMessageQueue.ts new file mode 100644 index 00000000000..12827d8bbc8 --- /dev/null +++ b/src/ToDeviceMessageQueue.ts @@ -0,0 +1,125 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "./logger"; +import { MatrixClient } from "./matrix"; +import { IndexedToDeviceBatch, ToDeviceBatch, ToDeviceBatchWithTxnId, ToDevicePayload } from "./models/ToDeviceMessage"; +import { MatrixScheduler } from "./scheduler"; + +const MAX_BATCH_SIZE = 20; + +/** + * Maintains a queue of outgoing to-device messages, sending them + * as soon as the homeserver is reachable. + */ +export class ToDeviceMessageQueue { + private sending = false; + private running = true; + private retryTimeout: number = null; + private retryAttempts = 0; + + constructor(private client: MatrixClient) { + } + + public start(): void { + this.running = true; + this.sendQueue(); + } + + public stop(): void { + this.running = false; + if (this.retryTimeout !== null) clearTimeout(this.retryTimeout); + this.retryTimeout = null; + } + + public async queueBatch(batch: ToDeviceBatch): Promise { + const batches: ToDeviceBatchWithTxnId[] = []; + for (let i = 0; i < batch.batch.length; i += MAX_BATCH_SIZE) { + batches.push({ + eventType: batch.eventType, + batch: batch.batch.slice(i, i + MAX_BATCH_SIZE), + txnId: this.client.makeTxnId(), + }); + } + + await this.client.store.saveToDeviceBatches(batches); + this.sendQueue(); + } + + public sendQueue = async (): Promise => { + if (this.retryTimeout !== null) clearTimeout(this.retryTimeout); + this.retryTimeout = null; + + if (this.sending || !this.running) return; + + logger.debug("Attempting to send queued to-device messages"); + + this.sending = true; + let headBatch; + try { + while (this.running) { + headBatch = await this.client.store.getOldestToDeviceBatch(); + if (headBatch === null) break; + await this.sendBatch(headBatch); + await this.client.store.removeToDeviceBatch(headBatch.id); + this.retryAttempts = 0; + } + + // Make sure we're still running after the async tasks: if not, stop. + if (!this.running) return; + + logger.debug("All queued to-device messages sent"); + } catch (e) { + ++this.retryAttempts; + // eslint-disable-next-line @typescript-eslint/naming-convention + // eslint-disable-next-line new-cap + const retryDelay = MatrixScheduler.RETRY_BACKOFF_RATELIMIT(null, this.retryAttempts, e); + if (retryDelay === -1) { + // the scheduler function doesn't differentiate between fatal errors and just getting + // bored and giving up for now + if (Math.floor(e.httpStatus / 100) === 4) { + logger.error("Fatal error when sending to-device message - dropping to-device batch!", e); + await this.client.store.removeToDeviceBatch(headBatch.id); + } else { + logger.info("Automatic retry limit reached for to-device messages."); + } + return; + } + + logger.info(`Failed to send batch of to-device messages. Will retry in ${retryDelay}ms`, e); + this.retryTimeout = setTimeout(this.sendQueue, retryDelay); + } finally { + this.sending = false; + } + }; + + /** + * Attempts to send a batch of to-device messages. + */ + private async sendBatch(batch: IndexedToDeviceBatch): Promise { + const contentMap: Record> = {}; + for (const item of batch.batch) { + if (!contentMap[item.userId]) { + contentMap[item.userId] = {}; + } + contentMap[item.userId][item.deviceId] = item.payload; + } + + logger.info(`Sending batch of ${batch.batch.length} to-device messages with ID ${batch.id}`); + + await this.client.sendToDevice(batch.eventType, contentMap, batch.txnId); + } +} diff --git a/src/client.ts b/src/client.ts index 03f57324993..5899146871f 100644 --- a/src/client.ts +++ b/src/client.ts @@ -41,9 +41,11 @@ import { sleep } from './utils'; import { Direction, EventTimeline } from "./models/event-timeline"; import { IActionsObject, PushProcessor } from "./pushprocessor"; import { AutoDiscovery, AutoDiscoveryAction } from "./autodiscovery"; +import { IEncryptAndSendToDevicesResult } from "./crypto"; import * as olmlib from "./crypto/olmlib"; import { decodeBase64, encodeBase64 } from "./crypto/olmlib"; -import { IExportedDevice as IOlmDevice } from "./crypto/OlmDevice"; +import { IExportedDevice as IExportedOlmDevice } from "./crypto/OlmDevice"; +import { IOlmDevice } from "./crypto/algorithms/megolm"; import { TypedReEmitter } from './ReEmitter'; import { IRoomEncryption, RoomList } from './crypto/RoomList'; import { logger } from './logger'; @@ -202,6 +204,8 @@ import { MSC3575SlidingSyncRequest, MSC3575SlidingSyncResponse, SlidingSync } fr import { SlidingSyncSdk } from "./sliding-sync-sdk"; import { Thread, THREAD_RELATION_TYPE } from "./models/thread"; import { MBeaconInfoEventContent, M_BEACON_INFO } from "./@types/beacon"; +import { ToDeviceMessageQueue } from "./ToDeviceMessageQueue"; +import { ToDeviceBatch } from "./models/ToDeviceMessage"; export type Store = IStore; @@ -214,7 +218,7 @@ const CAPABILITIES_CACHE_MS = 21600000; // 6 hours - an arbitrary value const TURN_CHECK_INTERVAL = 10 * 60 * 1000; // poll for turn credentials every 10 minutes interface IExportedDevice { - olmDevice: IOlmDevice; + olmDevice: IExportedOlmDevice; userId: string; deviceId: string; } @@ -955,13 +959,14 @@ export class MatrixClient extends TypedEventEmitter; - protected exportedOlmDeviceToImport: IOlmDevice; + protected exportedOlmDeviceToImport: IExportedOlmDevice; protected txnCtr = 0; protected mediaHandler = new MediaHandler(this); protected sessionId: string; protected pendingEventEncryption = new Map>(); private useE2eForGroupCall = true; + private toDeviceMessageQueue: ToDeviceMessageQueue; constructor(opts: IMatrixClientCreateOpts) { super(); @@ -1061,6 +1066,8 @@ export class MatrixClient extends TypedEventEmitterexplicitly attempts to retry their lost connection. + * Will also retry any outbound to-device messages currently in the queue to be sent + * (retries of regular outgoing events are handled separately, per-event). * @return {boolean} True if this resulted in a request being retried. */ public retryImmediately(): boolean { + // don't await for this promise: we just want to kick it off + this.toDeviceMessageQueue.sendQueue(); return this.syncApi.retryImmediately(); } @@ -2635,6 +2650,30 @@ export class MatrixClient extends TypedEventEmitter} Promise which + * resolves once the message has been encrypted and sent to the given + * userDeviceMap, and returns the { contentMap, deviceInfoByDeviceId } + * of the successfully sent messages. + */ + public encryptAndSendToDevices( + userDeviceInfoArr: IOlmDevice[], + payload: object, + ): Promise { + if (!this.crypto) { + throw new Error("End-to-End encryption disabled"); + } + return this.crypto.encryptAndSendToDevices(userDeviceInfoArr, payload); + } + /** * Forces the current outbound group session to be discarded such * that another one will be created next time an event is sent. @@ -3591,7 +3630,7 @@ export class MatrixClient extends TypedEventEmitter { + // also kick the to-device queue to retry + this.toDeviceMessageQueue.sendQueue(); + this.updatePendingEventStatus(room, event, EventStatus.SENDING); return this.encryptAndSendEvent(room, event); } @@ -8786,7 +8828,10 @@ export class MatrixClient extends TypedEventEmitter>} contentMap @@ -8818,6 +8863,17 @@ export class MatrixClient extends TypedEventEmitter { + return this.toDeviceMessageQueue.queueBatch(batch); + } + /** * Get the third party protocols that can be reached using * this HS diff --git a/src/crypto/algorithms/megolm.ts b/src/crypto/algorithms/megolm.ts index d479383351c..0df456f0a8a 100644 --- a/src/crypto/algorithms/megolm.ts +++ b/src/crypto/algorithms/megolm.ts @@ -606,17 +606,15 @@ class MegolmEncryption extends EncryptionAlgorithm { return this.crypto.encryptAndSendToDevices( userDeviceMap, payload, - ).then(({ contentMap, deviceInfoByUserIdAndDeviceId }) => { + ).then(({ toDeviceBatch, deviceInfoByUserIdAndDeviceId }) => { // store that we successfully uploaded the keys of the current slice - for (const userId of Object.keys(contentMap)) { - for (const deviceId of Object.keys(contentMap[userId])) { - session.markSharedWithDevice( - userId, - deviceId, - deviceInfoByUserIdAndDeviceId.get(userId).get(deviceId).getIdentityKey(), - chainIndex, - ); - } + for (const msg of toDeviceBatch.batch) { + session.markSharedWithDevice( + msg.userId, + msg.deviceId, + deviceInfoByUserIdAndDeviceId.get(msg.userId).get(msg.deviceId).getIdentityKey(), + chainIndex, + ); } }).catch((error) => { logger.error("failed to encryptAndSendToDevices", error); diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 126fc89e976..ff5ab43fea9 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -23,6 +23,7 @@ limitations under the License. import anotherjson from "another-json"; +import { EventType } from "../@types/event"; import { TypedReEmitter } from '../ReEmitter'; import { logger } from '../logger'; import { IExportedDevice, OlmDevice } from "./OlmDevice"; @@ -69,6 +70,7 @@ import { IStore } from "../store"; import { Room, RoomEvent } from "../models/room"; import { RoomMember, RoomMemberEvent } from "../models/room-member"; import { EventStatus, IClearEvent, IEvent, MatrixEvent, MatrixEventEvent } from "../models/event"; +import { ToDeviceBatch } from "../models/ToDeviceMessage"; import { ClientEvent, ICrossSigningKey, @@ -210,8 +212,8 @@ export interface IEncryptedContent { } /* eslint-enable camelcase */ -interface IEncryptAndSendToDevicesResult { - contentMap: Record>; +export interface IEncryptAndSendToDevicesResult { + toDeviceBatch: ToDeviceBatch; deviceInfoByUserIdAndDeviceId: Map>; } @@ -3115,106 +3117,91 @@ export class Crypto extends TypedEventEmitter} Promise which * resolves once the message has been encrypted and sent to the given * userDeviceMap, and returns the { contentMap, deviceInfoByDeviceId } * of the successfully sent messages. */ - public encryptAndSendToDevices( + public async encryptAndSendToDevices( userDeviceInfoArr: IOlmDevice[], payload: object, ): Promise { - const contentMap: Record> = {}; + const toDeviceBatch: ToDeviceBatch = { + eventType: EventType.RoomMessageEncrypted, + batch: [], + }; const deviceInfoByUserIdAndDeviceId = new Map>(); - const promises: Promise[] = []; - for (const { userId, deviceInfo } of userDeviceInfoArr) { - const deviceId = deviceInfo.deviceId; - const encryptedContent: IEncryptedContent = { - algorithm: olmlib.OLM_ALGORITHM, - sender_key: this.olmDevice.deviceCurve25519Key, - ciphertext: {}, - }; + try { + await Promise.all(userDeviceInfoArr.map(async ({ userId, deviceInfo }) => { + const deviceId = deviceInfo.deviceId; + const encryptedContent: IEncryptedContent = { + algorithm: olmlib.OLM_ALGORITHM, + sender_key: this.olmDevice.deviceCurve25519Key, + ciphertext: {}, + }; - // Assign to temp value to make type-checking happy - let userIdDeviceInfo = deviceInfoByUserIdAndDeviceId.get(userId); + // Assign to temp value to make type-checking happy + let userIdDeviceInfo = deviceInfoByUserIdAndDeviceId.get(userId); - if (userIdDeviceInfo === undefined) { - userIdDeviceInfo = new Map(); - deviceInfoByUserIdAndDeviceId.set(userId, userIdDeviceInfo); - } + if (userIdDeviceInfo === undefined) { + userIdDeviceInfo = new Map(); + deviceInfoByUserIdAndDeviceId.set(userId, userIdDeviceInfo); + } - // We hold by reference, this updates deviceInfoByUserIdAndDeviceId[userId] - userIdDeviceInfo.set(deviceId, deviceInfo); + // We hold by reference, this updates deviceInfoByUserIdAndDeviceId[userId] + userIdDeviceInfo.set(deviceId, deviceInfo); - if (!contentMap[userId]) { - contentMap[userId] = {}; - } - contentMap[userId][deviceId] = encryptedContent; + toDeviceBatch.batch.push({ + userId, + deviceId, + payload: encryptedContent, + }); - promises.push( - olmlib.ensureOlmSessionsForDevices( + await olmlib.ensureOlmSessionsForDevices( this.olmDevice, this.baseApis, { [userId]: [deviceInfo] }, - ).then(() => - olmlib.encryptMessageForDevice( - encryptedContent.ciphertext, - this.userId, - this.deviceId, - this.olmDevice, - userId, - deviceInfo, - payload, - ), - ), - ); - } + ); + await olmlib.encryptMessageForDevice( + encryptedContent.ciphertext, + this.userId, + this.deviceId, + this.olmDevice, + userId, + deviceInfo, + payload, + ); + })); - return Promise.all(promises).then(() => { // prune out any devices that encryptMessageForDevice could not encrypt for, // in which case it will have just not added anything to the ciphertext object. // There's no point sending messages to devices if we couldn't encrypt to them, // since that's effectively a blank message. - for (const userId of Object.keys(contentMap)) { - for (const deviceId of Object.keys(contentMap[userId])) { - if (Object.keys(contentMap[userId][deviceId].ciphertext).length === 0) { - logger.log(`No ciphertext for device ${userId}:${deviceId}: pruning`); - delete contentMap[userId][deviceId]; - } - } - // No devices left for that user? Strip that too. - if (Object.keys(contentMap[userId]).length === 0) { - logger.log(`Pruned all devices for user ${userId}`); - delete contentMap[userId]; + toDeviceBatch.batch = toDeviceBatch.batch.filter(msg => { + if (Object.keys(msg.payload.ciphertext).length > 0) { + return true; + } else { + logger.log(`No ciphertext for device ${msg.userId}:${msg.deviceId}: pruning`); + return false; } - } + }); - // Is there anything left? - if (Object.keys(contentMap).length === 0) { - logger.log("No users left to send to: aborting"); - return; + try { + await this.baseApis.queueToDevice(toDeviceBatch); + return { toDeviceBatch, deviceInfoByUserIdAndDeviceId }; + } catch (e) { + logger.error("sendToDevice failed", e); + throw e; } - - return this.baseApis.sendToDevice("m.room.encrypted", contentMap).then( - (response) => ({ contentMap, deviceInfoByUserIdAndDeviceId }), - ).catch(error => { - logger.error("sendToDevice failed", error); - throw error; - }); - }).catch(error => { - logger.error("encryptAndSendToDevices promises failed", error); - throw error; - }); + } catch (e) { + logger.error("encryptAndSendToDevices promises failed", e); + throw e; + } } private onMembership = (event: MatrixEvent, member: RoomMember, oldMembership?: string) => { diff --git a/src/models/ToDeviceMessage.ts b/src/models/ToDeviceMessage.ts new file mode 100644 index 00000000000..8efc3ed4e31 --- /dev/null +++ b/src/models/ToDeviceMessage.ts @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +export type ToDevicePayload = Record; + +export interface ToDeviceMessage { + userId: string; + deviceId: string; + payload: ToDevicePayload; +} + +export interface ToDeviceBatch { + eventType: string; + batch: ToDeviceMessage[]; +} + +// Only used internally +export interface ToDeviceBatchWithTxnId extends ToDeviceBatch { + txnId: string; +} + +// Only used internally +export interface IndexedToDeviceBatch extends ToDeviceBatchWithTxnId { + id: number; +} diff --git a/src/models/room.ts b/src/models/room.ts index eaaa8f9efdb..2992db58735 100644 --- a/src/models/room.ts +++ b/src/models/room.ts @@ -72,7 +72,7 @@ function synthesizeReceipt(userId: string, event: MatrixEvent, receiptType: Rece }, }, }, - type: "m.receipt", + type: EventType.Receipt, room_id: event.getRoomId(), }); } @@ -2423,9 +2423,9 @@ export class Room extends TypedEventEmitter */ public addEphemeralEvents(events: MatrixEvent[]): void { for (const event of events) { - if (event.getType() === 'm.typing') { + if (event.getType() === EventType.Typing) { this.currentState.setTypingEvent(event); - } else if (event.getType() === 'm.receipt') { + } else if (event.getType() === EventType.Receipt) { this.addReceipt(event); } // else ignore - life is too short for us to care about these events } diff --git a/src/scheduler.ts b/src/scheduler.ts index d0249b6cc02..271982b745a 100644 --- a/src/scheduler.ts +++ b/src/scheduler.ts @@ -57,7 +57,7 @@ export class MatrixScheduler { * failure was due to a rate limited request, the time specified in the error is * waited before being retried. * @param {MatrixEvent} event - * @param {Number} attempts + * @param {Number} attempts Number of attempts that have been made, including the one that just failed (ie. starting at 1) * @param {MatrixError} err * @return {Number} * @see module:scheduler~retryAlgorithm diff --git a/src/sliding-sync.ts b/src/sliding-sync.ts index 9d19ddfed42..5254a077dcf 100644 --- a/src/sliding-sync.ts +++ b/src/sliding-sync.ts @@ -44,6 +44,9 @@ export interface MSC3575Filter { is_invite?: boolean; is_tombstoned?: boolean; room_name_like?: string; + room_types?: string[]; + not_room_types?: string[]; + spaces?: string[]; } /** @@ -602,7 +605,7 @@ export class SlidingSync extends TypedEventEmitter; // type : content @@ -57,21 +57,21 @@ export interface IStore { setSyncToken(token: string): void; /** - * No-op. - * @param {Room} room + * Store the given room. + * @param {Room} room The room to be stored. All properties must be stored. */ storeRoom(room: Room): void; /** - * No-op. - * @param {string} roomId - * @return {null} + * Retrieve a room by its' room ID. + * @param {string} roomId The room ID. + * @return {Room} The room or null. */ getRoom(roomId: string): Room | null; /** - * No-op. - * @return {Array} An empty array. + * Retrieve all known rooms. + * @return {Room[]} A list of rooms, which may be empty. */ getRooms(): Room[]; @@ -82,35 +82,36 @@ export interface IStore { removeRoom(roomId: string): void; /** - * No-op. - * @return {Array} An empty array. + * Retrieve a summary of all the rooms. + * @return {RoomSummary[]} A summary of each room. */ getRoomSummaries(): RoomSummary[]; /** - * No-op. - * @param {User} user + * Store a User. + * @param {User} user The user to store. */ storeUser(user: User): void; /** - * No-op. - * @param {string} userId - * @return {null} + * Retrieve a User by its' user ID. + * @param {string} userId The user ID. + * @return {User} The user or null. */ getUser(userId: string): User | null; /** - * No-op. - * @return {User[]} + * Retrieve all known users. + * @return {User[]} A list of users, which may be empty. */ getUsers(): User[]; /** - * No-op. - * @param {Room} room - * @param {number} limit - * @return {Array} + * Retrieve scrollback for this room. + * @param {Room} room The matrix room + * @param {number} limit The max number of old events to retrieve. + * @return {Array} An array of objects which will be at most 'limit' + * length and at least 0. The objects are the raw event JSON. */ scrollback(room: Room, limit: number): MatrixEvent[]; @@ -209,8 +210,23 @@ export interface IStore { */ deleteAllData(): Promise; + /** + * Returns the out-of-band membership events for this room that + * were previously loaded. + * @param {string} roomId + * @returns {event[]} the events, potentially an empty array if OOB loading didn't yield any new members + * @returns {null} in case the members for this room haven't been stored yet + */ getOutOfBandMembers(roomId: string): Promise; + /** + * Stores the out-of-band membership events for this room. Note that + * it still makes sense to store an empty array as the OOB status for the room is + * marked as fetched, and getOutOfBandMembers will return an empty array instead of null + * @param {string} roomId + * @param {event[]} membershipEvents the membership events to store + * @returns {Promise} when all members have been stored + */ setOutOfBandMembers(roomId: string, membershipEvents: IStateEventWithRoomId[]): Promise; clearOutOfBandMembers(roomId: string): Promise; @@ -222,4 +238,19 @@ export interface IStore { getPendingEvents(roomId: string): Promise[]>; setPendingEvents(roomId: string, events: Partial[]): Promise; + + /** + * Stores batches of outgoing to-device messages + */ + saveToDeviceBatches(batch: ToDeviceBatchWithTxnId[]): Promise; + + /** + * Fetches the oldest batch of to-device messages in the queue + */ + getOldestToDeviceBatch(): Promise; + + /** + * Removes a specific batch of to-device messages from the queue + */ + removeToDeviceBatch(id: number): Promise; } diff --git a/src/store/indexeddb-backend.ts b/src/store/indexeddb-backend.ts index 83470d72a19..93d1cb3ab19 100644 --- a/src/store/indexeddb-backend.ts +++ b/src/store/indexeddb-backend.ts @@ -16,6 +16,7 @@ limitations under the License. import { ISavedSync } from "./index"; import { IEvent, IStartClientOpts, IStateEventWithRoomId, ISyncResponse } from ".."; +import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage"; export interface IIndexedDBBackend { connect(): Promise; @@ -31,6 +32,9 @@ export interface IIndexedDBBackend { getUserPresenceEvents(): Promise; getClientOptions(): Promise; storeClientOptions(options: IStartClientOpts): Promise; + saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise; + getOldestToDeviceBatch(): Promise; + removeToDeviceBatch(id: number): Promise; } export type UserTuple = [userId: string, presenceEvent: Partial]; diff --git a/src/store/indexeddb-local-backend.ts b/src/store/indexeddb-local-backend.ts index 178931ff857..bd646c37202 100644 --- a/src/store/indexeddb-local-backend.ts +++ b/src/store/indexeddb-local-backend.ts @@ -21,8 +21,9 @@ import { logger } from '../logger'; import { IStartClientOpts, IStateEventWithRoomId } from ".."; import { ISavedSync } from "./index"; import { IIndexedDBBackend, UserTuple } from "./indexeddb-backend"; +import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage"; -const VERSION = 3; +const VERSION = 4; function createDatabase(db: IDBDatabase): void { // Make user store, clobber based on user ID. (userId property of User objects) @@ -49,6 +50,10 @@ function upgradeSchemaV3(db: IDBDatabase): void { { keyPath: ["clobber"] }); } +function upgradeSchemaV4(db: IDBDatabase): void { + db.createObjectStore("to_device_queue", { autoIncrement: true }); +} + /** * Helper method to collect results from a Cursor and promiseify it. * @param {ObjectStore|Index} store The store to perform openCursor on. @@ -112,7 +117,7 @@ function reqAsPromise(req: IDBRequest): Promise { }); } -function reqAsCursorPromise(req: IDBRequest): Promise { +function reqAsCursorPromise(req: IDBRequest): Promise { return reqAsEventPromise(req).then((event) => req.result); } @@ -177,6 +182,9 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { if (oldVersion < 3) { upgradeSchemaV3(db); } + if (oldVersion < 4) { + upgradeSchemaV4(db); + } // Expand as needed. }; @@ -561,4 +569,36 @@ export class LocalIndexedDBStoreBackend implements IIndexedDBBackend { }); // put == UPSERT await txnAsPromise(txn); } + + public async saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise { + const txn = this.db.transaction(["to_device_queue"], "readwrite"); + const store = txn.objectStore("to_device_queue"); + for (const batch of batches) { + store.add(batch); + } + await txnAsPromise(txn); + } + + public async getOldestToDeviceBatch(): Promise { + const txn = this.db.transaction(["to_device_queue"], "readonly"); + const store = txn.objectStore("to_device_queue"); + const cursor = await reqAsCursorPromise(store.openCursor()); + if (!cursor) return null; + + const resultBatch = cursor.value as ToDeviceBatchWithTxnId; + + return { + id: cursor.key as number, + txnId: resultBatch.txnId, + eventType: resultBatch.eventType, + batch: resultBatch.batch, + }; + } + + public async removeToDeviceBatch(id: number): Promise { + const txn = this.db.transaction(["to_device_queue"], "readwrite"); + const store = txn.objectStore("to_device_queue"); + store.delete(id); + await txnAsPromise(txn); + } } diff --git a/src/store/indexeddb-remote-backend.ts b/src/store/indexeddb-remote-backend.ts index 9c06105a1c5..67ab2ccd293 100644 --- a/src/store/indexeddb-remote-backend.ts +++ b/src/store/indexeddb-remote-backend.ts @@ -20,6 +20,7 @@ import { ISavedSync } from "./index"; import { IStartClientOpts } from "../client"; import { IStateEventWithRoomId, ISyncResponse } from ".."; import { IIndexedDBBackend, UserTuple } from "./indexeddb-backend"; +import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage"; export class RemoteIndexedDBStoreBackend implements IIndexedDBBackend { private worker: Worker; @@ -133,6 +134,18 @@ export class RemoteIndexedDBStoreBackend implements IIndexedDBBackend { return this.doCmd('getUserPresenceEvents'); } + public async saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise { + return this.doCmd('saveToDeviceBatches', [batches]); + } + + public async getOldestToDeviceBatch(): Promise { + return this.doCmd('getOldestToDeviceBatch'); + } + + public async removeToDeviceBatch(id: number): Promise { + return this.doCmd('removeToDeviceBatch', [id]); + } + private ensureStarted(): Promise { if (this.startPromise === null) { this.worker = this.workerFactory(); diff --git a/src/store/indexeddb-store-worker.ts b/src/store/indexeddb-store-worker.ts index 0d37dbce935..ced77696176 100644 --- a/src/store/indexeddb-store-worker.ts +++ b/src/store/indexeddb-store-worker.ts @@ -103,6 +103,15 @@ export class IndexedDBStoreWorker { case 'storeClientOptions': prom = this.backend.storeClientOptions(msg.args[0]); break; + case 'saveToDeviceBatches': + prom = this.backend.saveToDeviceBatches(msg.args[0]); + break; + case 'getOldestToDeviceBatch': + prom = this.backend.getOldestToDeviceBatch(); + break; + case 'removeToDeviceBatch': + prom = this.backend.removeToDeviceBatch(msg.args[0]); + break; } if (prom === undefined) { diff --git a/src/store/indexeddb.ts b/src/store/indexeddb.ts index 09a85fd1b54..44f684bdf8f 100644 --- a/src/store/indexeddb.ts +++ b/src/store/indexeddb.ts @@ -27,6 +27,7 @@ import { IIndexedDBBackend } from "./indexeddb-backend"; import { ISyncResponse } from "../sync-accumulator"; import { TypedEventEmitter } from "../models/typed-event-emitter"; import { IStateEventWithRoomId } from "../@types/search"; +import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage"; /** * This is an internal module. See {@link IndexedDBStore} for the public class. @@ -351,6 +352,18 @@ export class IndexedDBStore extends MemoryStore { this.localStorage.removeItem(pendingEventsKey(roomId)); } } + + public saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise { + return this.backend.saveToDeviceBatches(batches); + } + + public getOldestToDeviceBatch(): Promise { + return this.backend.getOldestToDeviceBatch(); + } + + public removeToDeviceBatch(id: number): Promise { + return this.backend.removeToDeviceBatch(id); + } } /** diff --git a/src/store/memory.ts b/src/store/memory.ts index cb49e425fdb..0ed43a5b5ac 100644 --- a/src/store/memory.ts +++ b/src/store/memory.ts @@ -30,6 +30,7 @@ import { ISavedSync, IStore } from "./index"; import { RoomSummary } from "../models/room-summary"; import { ISyncResponse } from "../sync-accumulator"; import { IStateEventWithRoomId } from "../@types/search"; +import { IndexedToDeviceBatch, ToDeviceBatchWithTxnId } from "../models/ToDeviceMessage"; function isValidFilterId(filterId: string): boolean { const isValidStr = typeof filterId === "string" && @@ -64,6 +65,8 @@ export class MemoryStore implements IStore { private oobMembers: Record = {}; // roomId: [member events] private pendingEvents: { [roomId: string]: Partial[] } = {}; private clientOptions = {}; + private pendingToDeviceBatches: IndexedToDeviceBatch[] = []; + private nextToDeviceBatchId = 0; constructor(opts: IOpts = {}) { this.localStorage = opts.localStorage; @@ -429,4 +432,26 @@ export class MemoryStore implements IStore { public async setPendingEvents(roomId: string, events: Partial[]): Promise { this.pendingEvents[roomId] = events; } + + public saveToDeviceBatches(batches: ToDeviceBatchWithTxnId[]): Promise { + for (const batch of batches) { + this.pendingToDeviceBatches.push({ + id: this.nextToDeviceBatchId++, + eventType: batch.eventType, + txnId: batch.txnId, + batch: batch.batch, + }); + } + return Promise.resolve(); + } + + public async getOldestToDeviceBatch(): Promise { + if (this.pendingToDeviceBatches.length === 0) return null; + return this.pendingToDeviceBatches[0]; + } + + public removeToDeviceBatch(id: number): Promise { + this.pendingToDeviceBatches = this.pendingToDeviceBatches.filter(batch => batch.id !== id); + return Promise.resolve(); + } } diff --git a/src/store/stub.ts b/src/store/stub.ts index c9fc57055fd..eb988a9733b 100644 --- a/src/store/stub.ts +++ b/src/store/stub.ts @@ -28,6 +28,7 @@ import { ISavedSync, IStore } from "./index"; import { RoomSummary } from "../models/room-summary"; import { ISyncResponse } from "../sync-accumulator"; import { IStateEventWithRoomId } from "../@types/search"; +import { IndexedToDeviceBatch, ToDeviceBatch } from "../models/ToDeviceMessage"; /** * Construct a stub store. This does no-ops on most store methods. @@ -270,4 +271,16 @@ export class StubStore implements IStore { public setPendingEvents(roomId: string, events: Partial[]): Promise { return Promise.resolve(); } + + public async saveToDeviceBatches(batch: ToDeviceBatch[]): Promise { + return Promise.resolve(); + } + + public getOldestToDeviceBatch(): Promise { + return Promise.resolve(null); + } + + public async removeToDeviceBatch(id: number): Promise { + return Promise.resolve(); + } } diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 526eaf76744..9ea6a6711d0 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -614,26 +614,26 @@ export class MatrixCall extends TypedEventEmitter feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.feeds.push(new CallFeed({ - client: this.client, - roomId: this.roomId, - userId, - stream, - purpose, - audioMuted, - videoMuted, - })); - this.emit(CallEvent.FeedsChanged, this.feeds); + if (this.getFeedByStreamId(stream.id)) { + logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); + return; } - logger.info(`Call ${this.callId} Pushed remote stream (id="${ - stream.id}", active="${stream.active}", purpose=${purpose})`); + this.feeds.push(new CallFeed({ + client: this.client, + roomId: this.roomId, + userId, + stream, + purpose, + audioMuted, + videoMuted, + })); + this.emit(CallEvent.FeedsChanged, this.feeds); + + logger.info( + `Call ${this.callId} pushed remote stream (id="${stream.id}", ` + + `active="${stream.active}", purpose=${purpose})`, + ); } /** @@ -655,25 +655,23 @@ export class MatrixCall extends TypedEventEmitter feed.purpose === purpose); - if (existingFeed) { - existingFeed.setNewStream(stream); - } else { - this.pushLocalFeed( - new CallFeed({ - client: this.client, - roomId: this.roomId, - audioMuted: false, - videoMuted: false, - userId, - stream, - purpose, - }), - addToPeerConnection, - ); - this.emit(CallEvent.FeedsChanged, this.feeds); + if (this.getFeedByStreamId(stream.id)) { + logger.warn(`Ignoring stream with id ${stream.id} because we already have a feed for it`); + return; } + + this.pushLocalFeed( + new CallFeed({ + client: this.client, + roomId: this.roomId, + audioMuted: false, + videoMuted: false, + userId, + stream, + purpose, + }), + addToPeerConnection, + ); } /** diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index c9133895212..44a568cea58 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -181,7 +181,9 @@ export class CallFeed extends TypedEventEmitter /** * Replaces the current MediaStream with a new one. - * This method should be only used by MatrixCall. + * The stream will be different and new stream as remore parties are + * concerned, but this can be used for convenience locally to set up + * volume listeners automatically on the new stream etc. * @param newStream new stream with which to replace the current one */ public setNewStream(newStream: MediaStream): void { diff --git a/yarn.lock b/yarn.lock index 437517387ed..9476b4d3406 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1307,7 +1307,6 @@ "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.12.tgz": version "3.2.12" - uid "0bce3c86f9d36a4984d3c3e07df1c3fb4c679bd9" resolved "https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.12.tgz#0bce3c86f9d36a4984d3c3e07df1c3fb4c679bd9" "@nicolo-ribaudo/chokidar-2@2.1.8-no-fsevents.3": @@ -1438,9 +1437,9 @@ "@octokit/openapi-types" "^12.10.0" "@sinclair/typebox@^0.24.1": - version "0.24.20" - resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.24.20.tgz#11a657875de6008622d53f56e063a6347c51a6dd" - integrity sha512-kVaO5aEFZb33nPMTZBxiPEkY+slxiPtqC7QX8f9B3eGOMBvEfuMfxp9DSTTCsRJPumPKjrge4yagyssO4q6qzQ== + version "0.24.26" + resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.24.26.tgz#84f9e8c1d93154e734a7947609a1dc7c7a81cc22" + integrity sha512-1ZVIyyS1NXDRVT8GjWD5jULjhDyM3IsIHef2VGUMdnWOlX2tkPjyEX/7K0TGSH2S8EaPhp1ylFdjSjUGQ+gecg== "@sinonjs/commons@^1.7.0": version "1.8.3" @@ -1581,9 +1580,9 @@ integrity sha512-eC4U9MlIcu2q0KQmXszyn5Akca/0jrQmwDRgpAMJai7qBWq4amIQhZyNau4VYGtCeALvW1/NtjzJJ567aZxfKA== "@types/node@*": - version "18.6.1" - resolved "https://registry.yarnpkg.com/@types/node/-/node-18.6.1.tgz#828e4785ccca13f44e2fb6852ae0ef11e3e20ba5" - integrity sha512-z+2vB6yDt1fNwKOeGbckpmirO+VBDuQqecXkgeIqDlaOtmKn6hPR/viQ8cxCfqLU4fTlvM3+YjM367TukWdxpg== + version "18.6.3" + resolved "https://registry.yarnpkg.com/@types/node/-/node-18.6.3.tgz#4e4a95b6fe44014563ceb514b2598b3e623d1c98" + integrity sha512-6qKpDtoaYLM+5+AFChLhHermMQxc3TOEFIDzrZLPRGHPrLEwqFkkT5Kx3ju05g6X7uDPazz3jHbKPX0KzCjntg== "@types/node@16": version "16.11.45" @@ -1610,6 +1609,11 @@ resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d" integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA== +"@types/sdp-transform@^2.4.5": + version "2.4.5" + resolved "https://registry.yarnpkg.com/@types/sdp-transform/-/sdp-transform-2.4.5.tgz#3167961e0a1a5265545e278627aa37c606003f53" + integrity sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg== + "@types/stack-utils@^2.0.0": version "2.0.1" resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c" @@ -4802,10 +4806,10 @@ matrix-events-sdk@^0.0.1-beta.7: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1-beta.7.tgz#5ffe45eba1f67cc8d7c2377736c728b322524934" integrity sha512-9jl4wtWanUFSy2sr2lCjErN/oC8KTAtaeaozJtrgot1JiQcEI4Rda9OLgQ7nLKaqb4Z/QUx/fR3XpDzm5Jy1JA== -matrix-mock-request@^2.1.0: - version "2.1.0" - resolved "https://registry.yarnpkg.com/matrix-mock-request/-/matrix-mock-request-2.1.0.tgz#86f5b0ef846865d0767d3a8e64f5bcd6ca94c178" - integrity sha512-Cjpl3yP6h0yu5GKG89m1XZXZlm69Kg/qHV41N/t6SrQsgcfM3Bfavqx9YrtG0UnuXGy4bBSZIe1QiWVeFPZw1A== +matrix-mock-request@^2.1.2: + version "2.1.2" + resolved "https://registry.yarnpkg.com/matrix-mock-request/-/matrix-mock-request-2.1.2.tgz#11e38ed1233dced88a6f2bfba1684d5c5b3aa2c2" + integrity sha512-/OXCIzDGSLPJ3fs+uzDrtaOHI/Sqp4iEuniRn31U8S06mPXbvAnXknHqJ4c6A/KVwJj/nPFbGXpK4wPM038I6A== dependencies: expect "^28.1.0" @@ -5816,6 +5820,11 @@ safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +sdp-transform@^2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/sdp-transform/-/sdp-transform-2.14.1.tgz#2bb443583d478dee217df4caa284c46b870d5827" + integrity sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw== + semver@7.0.0: version "7.0.0" resolved "https://registry.yarnpkg.com/semver/-/semver-7.0.0.tgz#5f3ca35761e47e05b206c6daff2cf814f0316b8e" From 88ce017333f979ad4206ca49544ef32d1a3f269d Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 9 Aug 2022 13:11:59 +0100 Subject: [PATCH 221/291] Fix return types of event sending functions (#2576) These had somehow got mixed up so the type check was failing. Nothing uses the response return type, so just return void. --- src/webrtc/call.ts | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 9ea6a6711d0..e1932cdf9c6 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -47,7 +47,6 @@ import { } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; -import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; import { DeviceInfo } from '../crypto/deviceinfo'; import { GroupCallUnknownDeviceError } from './groupCall'; @@ -2181,7 +2180,7 @@ export class MatrixCall extends TypedEventEmitter { + private async sendVoipEvent(eventType: string, content: object): Promise { const realContent = Object.assign({}, content, { version: VOIP_PROTO_VERSION, call_id: this.callId, @@ -2224,7 +2223,7 @@ export class MatrixCall extends TypedEventEmitter Date: Tue, 9 Aug 2022 13:43:02 +0100 Subject: [PATCH 222/291] Add basic creation / entering tests for group calls (#2575) * Add basic creation / entering tests for group calls * Missing space Co-authored-by: Robin * Assert more of the group call member event and also move call leaving to a finally so it doesn't leaving a call hagning if it fails. Co-authored-by: Robin --- spec/test-utils/webrtc.ts | 7 ++- spec/unit/webrtc/groupCall.spec.ts | 98 ++++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 spec/unit/webrtc/groupCall.spec.ts diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 8b59469126b..ed124ba7a14 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -58,9 +58,13 @@ class MockMediaStreamAudioSourceNode { connect() {} } +class MockAnalyser { + getFloatFrequencyData() { return 0.0; } +} + export class MockAudioContext { constructor() {} - createAnalyser() { return {}; } + createAnalyser() { return new MockAnalyser(); } createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } close() {} } @@ -154,4 +158,5 @@ export class MockMediaHandler { } stopUserMediaStream() { } hasAudioDevice() { return true; } + stopAllStreams() {} } diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts new file mode 100644 index 00000000000..36669ea151e --- /dev/null +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -0,0 +1,98 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { EventType, GroupCallIntent, GroupCallType, Room, RoomMember } from '../../../src'; +import { GroupCall } from "../../../src/webrtc/groupCall"; +import { MatrixClient } from "../../../src/client"; +import { MockAudioContext, MockMediaHandler } from '../../test-utils/webrtc'; + +const FAKE_SELF_USER_ID = "@me:test.dummy"; +const FAKE_SELF_DEVICE_ID = "AAAAAA"; +const FAKE_SELF_SESSION_ID = "1"; +const FAKE_ROOM_ID = "!fake:test.dummy"; + +describe('Group Call', function() { + beforeEach(function() { + // @ts-ignore Mock + global.AudioContext = MockAudioContext; + }); + + it("sends state event to room when creating", async () => { + const mockSendState = jest.fn(); + + const mockClient = { + sendStateEvent: mockSendState, + groupCallEventHandler: { + groupCalls: new Map(), + }, + } as unknown as MatrixClient; + + const room = new Room(FAKE_ROOM_ID, mockClient, FAKE_SELF_USER_ID); + const groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); + + await groupCall.create(); + + expect(mockSendState.mock.calls[0][0]).toEqual(FAKE_ROOM_ID); + expect(mockSendState.mock.calls[0][1]).toEqual(EventType.GroupCallPrefix); + expect(mockSendState.mock.calls[0][2]["m.type"]).toEqual(GroupCallType.Video); + expect(mockSendState.mock.calls[0][2]["m.intent"]).toEqual(GroupCallIntent.Prompt); + }); + + it("sends member state event to room on enter", async () => { + const mockSendState = jest.fn(); + const mockMediaHandler = new MockMediaHandler(); + + const mockClient = { + sendStateEvent: mockSendState, + groupCallEventHandler: { + groupCalls: new Map(), + }, + callEventHandler: { + calls: new Map(), + }, + mediaHandler: mockMediaHandler, + getMediaHandler: () => mockMediaHandler, + getUserId: () => FAKE_SELF_USER_ID, + getDeviceId: () => FAKE_SELF_DEVICE_ID, + getSessionId: () => FAKE_SELF_SESSION_ID, + emit: jest.fn(), + on: jest.fn(), + removeListener: jest.fn(), + } as unknown as MatrixClient; + + const room = new Room(FAKE_ROOM_ID, mockClient, FAKE_SELF_USER_ID); + const groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); + + room.currentState.members[FAKE_SELF_USER_ID] = { + userId: FAKE_SELF_USER_ID, + } as unknown as RoomMember; + + await groupCall.create(); + + try { + await groupCall.enter(); + + expect(mockSendState.mock.lastCall[0]).toEqual(FAKE_ROOM_ID); + expect(mockSendState.mock.lastCall[1]).toEqual(EventType.GroupCallMemberPrefix); + expect(mockSendState.mock.lastCall[2]['m.calls'].length).toEqual(1); + expect(mockSendState.mock.lastCall[2]['m.calls'][0]["m.call_id"]).toEqual(groupCall.groupCallId); + expect(mockSendState.mock.lastCall[2]['m.calls'][0]['m.devices'].length).toEqual(1); + expect(mockSendState.mock.lastCall[2]['m.calls'][0]['m.devices'][0].device_id).toEqual(FAKE_SELF_DEVICE_ID); + } finally { + groupCall.leave(); + } + }); +}); From 3334c01191bcd82b5243916284c9a08d08fd9795 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 9 Aug 2022 09:45:58 -0400 Subject: [PATCH 223/291] Support nested Matrix clients via the widget API (#2473) * WIP RoomWidgetClient * Wait for the widget API to become ready before backfilling * Add support for sending user-defined encrypted to-device messages This is a port of the same change from the robertlong/group-call branch. * Fix tests * Emit an event when the client receives TURN servers * Expose the method in MatrixClient * Override the encryptAndSendToDevices method * Add support for TURN servers in embedded mode and make calls mostly work * Don't put unclonable objects into VoIP events RoomWidget clients were unable to send m.call.candidate events, because the candidate objects were not clonable for use with postMessage. Converting such objects to their canonical JSON form before attempting to send them over the wire solves this. * Fix types * Fix more types * Fix lint * Upgrade matrix-widget-api * Save lockfile * Untangle dependencies to fix tests * Add some preliminary tests * Fix tests * Fix indirect export * Add more tests * Resolve TODOs * Add queueToDevice to RoomWidgetClient --- package.json | 2 + spec/unit/embedded.spec.ts | 264 ++++++++++++++++++++ src/ToDeviceMessageQueue.ts | 2 +- src/client.ts | 46 ++-- src/crypto/CrossSigning.ts | 2 +- src/crypto/EncryptionSetup.ts | 7 +- src/crypto/SecretStorage.ts | 5 +- src/crypto/index.ts | 23 ++ src/embedded.ts | 258 +++++++++++++++++++ src/matrix.ts | 65 +++-- src/models/beacon.ts | 2 +- src/models/thread.ts | 7 +- src/sliding-sync-sdk.ts | 123 ++++----- src/sync.ts | 15 +- src/utils.ts | 2 +- src/webrtc/call.ts | 42 ++-- src/webrtc/callEventHandler.ts | 2 + src/webrtc/groupCallEventHandler.ts | 4 +- yarn.lock | 370 ++++++++++++++++++++++++++-- 19 files changed, 1057 insertions(+), 184 deletions(-) create mode 100644 spec/unit/embedded.spec.ts create mode 100644 src/embedded.ts diff --git a/package.json b/package.json index 787e776bd68..bc6ab9b7075 100644 --- a/package.json +++ b/package.json @@ -61,6 +61,7 @@ "content-type": "^1.0.4", "loglevel": "^1.7.1", "matrix-events-sdk": "^0.0.1-beta.7", + "matrix-widget-api": "^1.0.0", "p-retry": "4", "qs": "^6.9.6", "request": "^2.88.2", @@ -101,6 +102,7 @@ "exorcist": "^2.0.0", "fake-indexeddb": "^4.0.0", "jest": "^28.0.0", + "jest-environment-jsdom": "^28.1.3", "jest-localstorage-mock": "^2.4.6", "jest-sonar-reporter": "^2.0.0", "jsdoc": "^3.6.6", diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts new file mode 100644 index 00000000000..edac107b976 --- /dev/null +++ b/spec/unit/embedded.spec.ts @@ -0,0 +1,264 @@ +/** + * @jest-environment jsdom + */ + +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// We have to use EventEmitter here to mock part of the matrix-widget-api +// project, which doesn't know about our TypeEventEmitter implementation at all +// eslint-disable-next-line no-restricted-imports +import { EventEmitter } from "events"; +import { MockedObject } from "jest-mock"; +import { + WidgetApi, + WidgetApiToWidgetAction, + MatrixCapabilities, + ITurnServer, +} from "matrix-widget-api"; + +import { createRoomWidgetClient } from "../../src/matrix"; +import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client"; +import { SyncState } from "../../src/sync"; +import { ICapabilities } from "../../src/embedded"; +import { MatrixEvent } from "../../src/models/event"; +import { ToDeviceBatch } from "../../src/models/ToDeviceMessage"; +import { DeviceInfo } from "../../src/crypto/deviceinfo"; + +class MockWidgetApi extends EventEmitter { + public start = jest.fn(); + public requestCapability = jest.fn(); + public requestCapabilities = jest.fn(); + public requestCapabilityToSendState = jest.fn(); + public requestCapabilityToReceiveState = jest.fn(); + public requestCapabilityToSendToDevice = jest.fn(); + public requestCapabilityToReceiveToDevice = jest.fn(); + public sendStateEvent = jest.fn(); + public sendToDevice = jest.fn(); + public readStateEvents = jest.fn(() => []); + public getTurnServers = jest.fn(() => []); + + public transport = { reply: jest.fn() }; +} + +describe("RoomWidgetClient", () => { + let widgetApi: MockedObject; + let client: MatrixClient; + + beforeEach(() => { + widgetApi = new MockWidgetApi() as unknown as MockedObject; + }); + + afterEach(() => { + client.stopClient(); + }); + + const makeClient = async (capabilities: ICapabilities): Promise => { + const baseUrl = "https://example.org"; + client = createRoomWidgetClient(widgetApi, capabilities, "!1:example.org", { baseUrl }); + expect(widgetApi.start).toHaveBeenCalled(); // needs to have been called early in order to not miss messages + widgetApi.emit("ready"); + await client.startClient(); + }; + + describe("state events", () => { + const event = new MatrixEvent({ + type: "org.example.foo", + event_id: "$sfkjfsksdkfsd", + room_id: "!1:example.org", + sender: "@alice:example.org", + state_key: "bar", + content: { hello: "world" }, + }).getEffectiveEvent(); + + it("sends", async () => { + await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar"); + await client.sendStateEvent("!1:example.org", "org.example.foo", { hello: "world" }, "bar"); + expect(widgetApi.sendStateEvent).toHaveBeenCalledWith("org.example.foo", "bar", { hello: "world" }); + }); + + it("refuses to send to other rooms", async () => { + await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar"); + await expect(client.sendStateEvent("!2:example.org", "org.example.foo", { hello: "world" }, "bar")) + .rejects.toBeDefined(); + }); + + it("receives", async () => { + await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); + + const emittedEvent = new Promise(resolve => client.once(ClientEvent.Event, resolve)); + const emittedSync = new Promise(resolve => client.once(ClientEvent.Sync, resolve)); + widgetApi.emit( + `action:${WidgetApiToWidgetAction.SendEvent}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.SendEvent}`, { detail: { data: event } }), + ); + + // The client should've emitted about the received event + expect((await emittedEvent).getEffectiveEvent()).toEqual(event); + expect(await emittedSync).toEqual(SyncState.Syncing); + // It should've also inserted the event into the room object + const room = client.getRoom("!1:example.org"); + expect(room.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + }); + + it("backfills", async () => { + widgetApi.readStateEvents.mockImplementation(async (eventType, limit, stateKey) => + eventType === "org.example.foo" && (limit ?? Infinity) > 0 && stateKey === "bar" + ? [event] + : [], + ); + + await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); + + const room = client.getRoom("!1:example.org"); + expect(room.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + }); + }); + + describe("to-device messages", () => { + const unencryptedContentMap = { + "@alice:example.org": { "*": { hello: "alice!" } }, + "@bob:example.org": { bobDesktop: { hello: "bob!" } }, + }; + + it("sends unencrypted (sendToDevice)", async () => { + await makeClient({ sendToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo"); + + await client.sendToDevice("org.example.foo", unencryptedContentMap); + expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", false, unencryptedContentMap); + }); + + it("sends unencrypted (queueToDevice)", async () => { + await makeClient({ sendToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo"); + + const batch: ToDeviceBatch = { + eventType: "org.example.foo", + batch: [ + { userId: "@alice:example.org", deviceId: "*", payload: { hello: "alice!" } }, + { userId: "@bob:example.org", deviceId: "bobDesktop", payload: { hello: "bob!" } }, + ], + }; + await client.queueToDevice(batch); + expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", false, unencryptedContentMap); + }); + + it("sends encrypted (encryptAndSendToDevices)", async () => { + await makeClient({ sendToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo"); + + const payload = { type: "org.example.foo", hello: "world" }; + await client.encryptAndSendToDevices( + [ + { userId: "@alice:example.org", deviceInfo: new DeviceInfo("aliceWeb") }, + { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobDesktop") }, + ], + payload, + ); + expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", true, { + "@alice:example.org": { aliceWeb: payload }, + "@bob:example.org": { bobDesktop: payload }, + }); + }); + + it.each([ + { encrypted: false, title: "unencrypted" }, + { encrypted: true, title: "encrypted" }, + ])("receives $title", async ({ encrypted }) => { + await makeClient({ receiveToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToReceiveToDevice).toHaveBeenCalledWith("org.example.foo"); + + const event = { + type: "org.example.foo", + sender: "@alice:example.org", + encrypted, + content: { hello: "world" }, + }; + + const emittedEvent = new Promise(resolve => client.once(ClientEvent.ToDeviceEvent, resolve)); + const emittedSync = new Promise(resolve => client.once(ClientEvent.Sync, resolve)); + widgetApi.emit( + `action:${WidgetApiToWidgetAction.SendToDevice}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.SendToDevice}`, { detail: { data: event } }), + ); + + expect((await emittedEvent).getEffectiveEvent()).toEqual({ + type: event.type, + sender: event.sender, + content: event.content, + }); + expect((await emittedEvent).isEncrypted()).toEqual(encrypted); + expect(await emittedSync).toEqual(SyncState.Syncing); + }); + }); + + it("gets TURN servers", async () => { + const server1: ITurnServer = { + uris: [ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp", + ], + username: "1443779631:@user:example.com", + password: "JlKfBy1QwLrO20385QyAtEyIv0=", + }; + const server2: ITurnServer = { + uris: [ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp", + ], + username: "1448999322:@user:example.com", + password: "hunter2", + }; + const clientServer1: IClientTurnServer = { + urls: server1.uris, + username: server1.username, + credential: server1.password, + }; + const clientServer2: IClientTurnServer = { + urls: server2.uris, + username: server2.username, + credential: server2.password, + }; + + let emitServer2: () => void; + const getServer2 = new Promise(resolve => emitServer2 = () => resolve(server2)); + widgetApi.getTurnServers.mockImplementation(async function* () { + yield server1; + yield await getServer2; + }); + + await makeClient({ turnServers: true }); + expect(widgetApi.requestCapability).toHaveBeenCalledWith(MatrixCapabilities.MSC3846TurnServers); + + // The first server should've arrived immediately + expect(client.getTurnServers()).toEqual([clientServer1]); + + // Subsequent servers arrive asynchronously and should emit an event + const emittedServer = new Promise(resolve => + client.once(ClientEvent.TurnServers, resolve), + ); + emitServer2(); + expect(await emittedServer).toEqual([clientServer2]); + expect(client.getTurnServers()).toEqual([clientServer2]); + }); +}); diff --git a/src/ToDeviceMessageQueue.ts b/src/ToDeviceMessageQueue.ts index 12827d8bbc8..66f471c415d 100644 --- a/src/ToDeviceMessageQueue.ts +++ b/src/ToDeviceMessageQueue.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { logger } from "./logger"; -import { MatrixClient } from "./matrix"; +import { MatrixClient } from "./client"; import { IndexedToDeviceBatch, ToDeviceBatch, ToDeviceBatchWithTxnId, ToDevicePayload } from "./models/ToDeviceMessage"; import { MatrixScheduler } from "./scheduler"; diff --git a/src/client.ts b/src/client.ts index 45e1cd17318..0aa83d7402c 100644 --- a/src/client.ts +++ b/src/client.ts @@ -72,6 +72,7 @@ import { CryptoEvent, CryptoEventHandlerMap, fixBackupKey, + ICryptoCallbacks, IBootstrapCrossSigningOpts, ICheckOwnCrossSigningTrustOpts, IMegolmSessionData, @@ -101,29 +102,9 @@ import { } from "./crypto/keybackup"; import { IIdentityServerProvider } from "./@types/IIdentityServerProvider"; import { MatrixScheduler } from "./scheduler"; -import { - IAuthData, - ICryptoCallbacks, - IMinimalEvent, - IRoomEvent, - IStateEvent, - NotificationCountType, - BeaconEvent, - BeaconEventHandlerMap, - RoomEvent, - RoomEventHandlerMap, - RoomMemberEvent, - RoomMemberEventHandlerMap, - RoomStateEvent, - RoomStateEventHandlerMap, - INotificationsResponse, - IFilterResponse, - ITagsResponse, - IStatusResponse, - IPushRule, - PushRuleActionName, - IAuthDict, -} from "./matrix"; +import { BeaconEvent, BeaconEventHandlerMap } from "./models/beacon"; +import { IAuthData, IAuthDict } from "./interactive-auth"; +import { IMinimalEvent, IRoomEvent, IStateEvent } from "./sync-accumulator"; import { CrossSigningKey, IAddSecretStorageKeyOpts, @@ -138,7 +119,9 @@ import { VerificationRequest } from "./crypto/verification/request/VerificationR import { VerificationBase as Verification } from "./crypto/verification/Base"; import * as ContentHelpers from "./content-helpers"; import { CrossSigningInfo, DeviceTrustLevel, ICacheCallbacks, UserTrustLevel } from "./crypto/CrossSigning"; -import { Room } from "./models/room"; +import { Room, NotificationCountType, RoomEvent, RoomEventHandlerMap } from "./models/room"; +import { RoomMemberEvent, RoomMemberEventHandlerMap } from "./models/room-member"; +import { RoomStateEvent, RoomStateEventHandlerMap } from "./models/room-state"; import { IAddThreePidOnlyBody, IBindThreePidBody, @@ -156,6 +139,10 @@ import { ISearchOpts, ISendEventResponse, IUploadOpts, + INotificationsResponse, + IFilterResponse, + ITagsResponse, + IStatusResponse, } from "./@types/requests"; import { EventType, @@ -185,7 +172,16 @@ import { } from "./@types/search"; import { ISynapseAdminDeactivateResponse, ISynapseAdminWhoisResponse } from "./@types/synapse"; import { IHierarchyRoom } from "./@types/spaces"; -import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules"; +import { + IPusher, + IPusherRequest, + IPushRule, + IPushRules, + PushRuleAction, + PushRuleActionName, + PushRuleKind, + RuleId, +} from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; import { diff --git a/src/crypto/CrossSigning.ts b/src/crypto/CrossSigning.ts index 4e5c9f452ec..6121d45fdd6 100644 --- a/src/crypto/CrossSigning.ts +++ b/src/crypto/CrossSigning.ts @@ -29,7 +29,7 @@ import { DeviceInfo } from "./deviceinfo"; import { SecretStorage } from "./SecretStorage"; import { ICrossSigningKey, ISignedKey, MatrixClient } from "../client"; import { OlmDevice } from "./OlmDevice"; -import { ICryptoCallbacks } from "../matrix"; +import { ICryptoCallbacks } from "."; import { ISignatures } from "../@types/signed"; import { CryptoStore } from "./store/base"; import { ISecretStorageKeyInfo } from "./api"; diff --git a/src/crypto/EncryptionSetup.ts b/src/crypto/EncryptionSetup.ts index 61ba34eaf99..456c1cac4e0 100644 --- a/src/crypto/EncryptionSetup.ts +++ b/src/crypto/EncryptionSetup.ts @@ -19,16 +19,15 @@ import { MatrixEvent } from "../models/event"; import { createCryptoStoreCacheCallbacks, ICacheCallbacks } from "./CrossSigning"; import { IndexedDBCryptoStore } from './store/indexeddb-crypto-store'; import { Method, PREFIX_UNSTABLE } from "../http-api"; -import { Crypto, IBootstrapCrossSigningOpts } from "./index"; +import { Crypto, ICryptoCallbacks, IBootstrapCrossSigningOpts } from "./index"; import { ClientEvent, - CrossSigningKeys, ClientEventHandlerMap, + CrossSigningKeys, ICrossSigningKey, - ICryptoCallbacks, ISignedKey, KeySignatures, -} from "../matrix"; +} from "../client"; import { ISecretStorageKeyInfo } from "./api"; import { IKeyBackupInfo } from "./keybackup"; import { TypedEventEmitter } from "../models/typed-event-emitter"; diff --git a/src/crypto/SecretStorage.ts b/src/crypto/SecretStorage.ts index 0eef2ee7d50..09bf467bb31 100644 --- a/src/crypto/SecretStorage.ts +++ b/src/crypto/SecretStorage.ts @@ -19,8 +19,9 @@ import * as olmlib from './olmlib'; import { encodeBase64 } from './olmlib'; import { randomString } from '../randomstring'; import { calculateKeyCheck, decryptAES, encryptAES, IEncryptedPayload } from './aes'; -import { ClientEvent, ICryptoCallbacks, MatrixEvent } from '../matrix'; -import { ClientEventHandlerMap, MatrixClient } from "../client"; +import { ICryptoCallbacks } from "."; +import { MatrixEvent } from "../models/event"; +import { ClientEvent, ClientEventHandlerMap, MatrixClient } from "../client"; import { IAddSecretStorageKeyOpts, ISecretStorageKeyInfo } from './api'; import { TypedEventEmitter } from '../models/typed-event-emitter'; diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 451d2d8c82a..e6d47378623 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -126,6 +126,29 @@ export interface IBootstrapCrossSigningOpts { authUploadDeviceSigningKeys?(makeRequest: (authData: any) => Promise<{}>): Promise; } +export interface ICryptoCallbacks { + getCrossSigningKey?: (keyType: string, pubKey: string) => Promise; + saveCrossSigningKeys?: (keys: Record) => void; + shouldUpgradeDeviceVerifications?: ( + users: Record + ) => Promise; + getSecretStorageKey?: ( + keys: {keys: Record}, name: string + ) => Promise<[string, Uint8Array] | null>; + cacheSecretStorageKey?: ( + keyId: string, keyInfo: ISecretStorageKeyInfo, key: Uint8Array + ) => void; + onSecretRequested?: ( + userId: string, deviceId: string, + requestId: string, secretName: string, deviceTrust: DeviceTrustLevel + ) => Promise; + getDehydrationKey?: ( + keyInfo: ISecretStorageKeyInfo, + checkFunc: (key: Uint8Array) => void, + ) => Promise; + getBackupKey?: () => Promise; +} + /* eslint-disable camelcase */ interface IRoomKey { room_id: string; diff --git a/src/embedded.ts b/src/embedded.ts new file mode 100644 index 00000000000..047cd0279c3 --- /dev/null +++ b/src/embedded.ts @@ -0,0 +1,258 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + WidgetApi, + WidgetApiToWidgetAction, + MatrixCapabilities, + IWidgetApiRequest, + IWidgetApiAcknowledgeResponseData, + ISendEventToWidgetActionRequest, + ISendToDeviceToWidgetActionRequest, +} from "matrix-widget-api"; + +import { ISendEventResponse } from "./@types/requests"; +import { EventType } from "./@types/event"; +import { logger } from "./logger"; +import { MatrixClient, ClientEvent, IMatrixClientCreateOpts, IStartClientOpts } from "./client"; +import { SyncApi, SyncState } from "./sync"; +import { SlidingSyncSdk } from "./sliding-sync-sdk"; +import { MatrixEvent } from "./models/event"; +import { User } from "./models/user"; +import { Room } from "./models/room"; +import { ToDeviceBatch } from "./models/ToDeviceMessage"; +import { DeviceInfo } from "./crypto/deviceinfo"; +import { IOlmDevice } from "./crypto/algorithms/megolm"; + +interface IStateEventRequest { + eventType: string; + stateKey?: string; +} + +export interface ICapabilities { + // TODO: Add fields for messages and other non-state events + + sendState?: IStateEventRequest[]; + receiveState?: IStateEventRequest[]; + + sendToDevice?: string[]; + receiveToDevice?: string[]; + + turnServers?: boolean; +} + +export class RoomWidgetClient extends MatrixClient { + private room: Room; + private widgetApiReady = new Promise(resolve => this.widgetApi.once("ready", resolve)); + private lifecycle: AbortController; + private syncState: SyncState | null = null; + + constructor( + private readonly widgetApi: WidgetApi, + private readonly capabilities: ICapabilities, + private readonly roomId: string, + opts: IMatrixClientCreateOpts, + ) { + super(opts); + + // Request capabilities for the functionality this client needs to support + this.capabilities.sendState?.forEach(({ eventType, stateKey }) => + this.widgetApi.requestCapabilityToSendState(eventType, stateKey), + ); + this.capabilities.receiveState?.forEach(({ eventType, stateKey }) => + this.widgetApi.requestCapabilityToReceiveState(eventType, stateKey), + ); + this.capabilities.sendToDevice?.forEach(eventType => + this.widgetApi.requestCapabilityToSendToDevice(eventType), + ); + this.capabilities.receiveToDevice?.forEach(eventType => + this.widgetApi.requestCapabilityToReceiveToDevice(eventType), + ); + if (this.capabilities.turnServers) { + this.widgetApi.requestCapability(MatrixCapabilities.MSC3846TurnServers); + } + + this.widgetApi.on(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); + this.widgetApi.on(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + + // Open communication with the host + this.widgetApi.start(); + } + + public async startClient(opts: IStartClientOpts = {}): Promise { + this.lifecycle = new AbortController(); + + // Create our own user object artificially (instead of waiting for sync) + // so it's always available, even if the user is not in any rooms etc. + const userId = this.getUserId(); + if (userId) { + this.store.storeUser(new User(userId)); + } + + // Even though we have no access token and cannot sync, the sync class + // still has some valuable helper methods that we make use of, so we + // instantiate it anyways + if (opts.slidingSync) { + this.syncApi = new SlidingSyncSdk(opts.slidingSync, this, opts); + } else { + this.syncApi = new SyncApi(this, opts); + } + + this.room = this.syncApi.createRoom(this.roomId); + this.store.storeRoom(this.room); + + await this.widgetApiReady; + + // Backfill the requested events + // We only get the most recent event for every type + state key combo, + // so it doesn't really matter what order we inject them in + await Promise.all( + this.capabilities.receiveState?.map(async ({ eventType, stateKey }) => { + const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey); + const events = rawEvents.map(rawEvent => new MatrixEvent(rawEvent)); + + await this.syncApi.injectRoomEvents(this.room, [], events); + events.forEach(event => { + this.emit(ClientEvent.Event, event); + logger.info(`Backfilled event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + }); + }) ?? [], + ); + this.setSyncState(SyncState.Prepared); + logger.info("Finished backfilling events"); + + // Watch for TURN servers, if requested + if (this.capabilities.turnServers) this.watchTurnServers(); + } + + public stopClient() { + this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); + this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + + super.stopClient(); + this.lifecycle.abort(); // Signal to other async tasks that the client has stopped + } + + public async sendStateEvent( + roomId: string, + eventType: string, + content: any, + stateKey = "", + ): Promise { + if (roomId !== this.roomId) throw new Error(`Can't send events to ${roomId}`); + return await this.widgetApi.sendStateEvent(eventType, stateKey, content); + } + + public async sendToDevice( + eventType: string, + contentMap: { [userId: string]: { [deviceId: string]: Record } }, + ): Promise<{}> { + await this.widgetApi.sendToDevice(eventType, false, contentMap); + return {}; + } + + public async queueToDevice({ eventType, batch }: ToDeviceBatch): Promise { + const contentMap: { [userId: string]: { [deviceId: string]: object } } = {}; + for (const { userId, deviceId, payload } of batch) { + if (!contentMap[userId]) contentMap[userId] = {}; + contentMap[userId][deviceId] = payload; + } + + await this.widgetApi.sendToDevice(eventType, false, contentMap); + } + + public async encryptAndSendToDevices( + userDeviceInfoArr: IOlmDevice[], + payload: object, + ): Promise { + const contentMap: { [userId: string]: { [deviceId: string]: object } } = {}; + for (const { userId, deviceInfo: { deviceId } } of userDeviceInfoArr) { + if (!contentMap[userId]) contentMap[userId] = {}; + contentMap[userId][deviceId] = payload; + } + + await this.widgetApi.sendToDevice((payload as { type: string }).type, true, contentMap); + } + + // Overridden since we get TURN servers automatically over the widget API, + // and this method would otherwise complain about missing an access token + public async checkTurnServers(): Promise { + return this.turnServers.length > 0; + } + + // Overridden since we 'sync' manually without the sync API + public getSyncState(): SyncState { + return this.syncState; + } + + private setSyncState(state: SyncState) { + const oldState = this.syncState; + this.syncState = state; + this.emit(ClientEvent.Sync, state, oldState); + } + + private async ack(ev: CustomEvent): Promise { + await this.widgetApi.transport.reply(ev.detail, {}); + } + + private onEvent = async (ev: CustomEvent) => { + ev.preventDefault(); + const event = new MatrixEvent(ev.detail.data); + await this.syncApi.injectRoomEvents(this.room, [], [event]); + this.emit(ClientEvent.Event, event); + this.setSyncState(SyncState.Syncing); + logger.info(`Received event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + await this.ack(ev); + }; + + private onToDevice = async (ev: CustomEvent) => { + ev.preventDefault(); + + const event = new MatrixEvent({ + type: ev.detail.data.type, + sender: ev.detail.data.sender, + content: ev.detail.data.content, + }); + // Mark the event as encrypted if it was, using fake contents and keys since those are unknown to us + if (ev.detail.data.encrypted) event.makeEncrypted(EventType.RoomMessageEncrypted, {}, "", ""); + + this.emit(ClientEvent.ToDeviceEvent, event); + this.setSyncState(SyncState.Syncing); + await this.ack(ev); + }; + + private async watchTurnServers() { + const servers = this.widgetApi.getTurnServers(); + const onClientStopped = () => servers.return(undefined); + this.lifecycle.signal.addEventListener("abort", onClientStopped); + + try { + for await (const server of servers) { + this.turnServers = [{ + urls: server.uris, + username: server.username, + credential: server.password, + }]; + this.emit(ClientEvent.TurnServers, this.turnServers); + logger.log(`Received TURN server: ${server.uris}`); + } + } catch (e) { + logger.warn("Error watching TURN servers", e); + } finally { + this.lifecycle.signal.removeEventListener("abort", onClientStopped); + } + } +} diff --git a/src/matrix.ts b/src/matrix.ts index 646f8879818..4fbd3077377 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -1,5 +1,5 @@ /* -Copyright 2015-2021 The Matrix.org Foundation C.I.C. +Copyright 2015-2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,14 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { WidgetApi } from "matrix-widget-api"; + import { MemoryCryptoStore } from "./crypto/store/memory-crypto-store"; import { MemoryStore } from "./store/memory"; import { MatrixScheduler } from "./scheduler"; import { MatrixClient, ICreateClientOpts } from "./client"; -import { DeviceTrustLevel } from "./crypto/CrossSigning"; -import { ISecretStorageKeyInfo } from "./crypto/api"; +import { RoomWidgetClient, ICapabilities } from "./embedded"; export * from "./client"; +export * from "./embedded"; export * from "./http-api"; export * from "./autodiscovery"; export * from "./sync-accumulator"; @@ -51,6 +53,7 @@ export * from './@types/requests'; export * from './@types/search'; export * from './models/room-summary'; export * as ContentHelpers from "./content-helpers"; +export type { ICryptoCallbacks } from "./crypto"; // used to be located here export { createNewMatrixCall } from "./webrtc/call"; export type { MatrixCall } from "./webrtc/call"; export { @@ -108,27 +111,17 @@ export function setCryptoStoreFactory(fac) { cryptoStoreFactory = fac; } -export interface ICryptoCallbacks { - getCrossSigningKey?: (keyType: string, pubKey: string) => Promise; - saveCrossSigningKeys?: (keys: Record) => void; - shouldUpgradeDeviceVerifications?: ( - users: Record - ) => Promise; - getSecretStorageKey?: ( - keys: {keys: Record}, name: string - ) => Promise<[string, Uint8Array] | null>; - cacheSecretStorageKey?: ( - keyId: string, keyInfo: ISecretStorageKeyInfo, key: Uint8Array - ) => void; - onSecretRequested?: ( - userId: string, deviceId: string, - requestId: string, secretName: string, deviceTrust: DeviceTrustLevel - ) => Promise; - getDehydrationKey?: ( - keyInfo: ISecretStorageKeyInfo, - checkFunc: (key: Uint8Array) => void, - ) => Promise; - getBackupKey?: () => Promise; +function amendClientOpts(opts: ICreateClientOpts | string): ICreateClientOpts { + if (typeof opts === "string") opts = { baseUrl: opts }; + + opts.request = opts.request ?? requestInstance; + opts.store = opts.store ?? new MemoryStore({ + localStorage: global.localStorage, + }); + opts.scheduler = opts.scheduler ?? new MatrixScheduler(); + opts.cryptoStore = opts.cryptoStore ?? cryptoStoreFactory(); + + return opts; } /** @@ -154,19 +147,17 @@ export interface ICryptoCallbacks { * @see {@link module:client.MatrixClient} for the full list of options for * opts. */ -export function createClient(opts: ICreateClientOpts | string) { - if (typeof opts === "string") { - opts = { - "baseUrl": opts, - }; - } - opts.request = opts.request || requestInstance; - opts.store = opts.store || new MemoryStore({ - localStorage: global.localStorage, - }); - opts.scheduler = opts.scheduler || new MatrixScheduler(); - opts.cryptoStore = opts.cryptoStore || cryptoStoreFactory(); - return new MatrixClient(opts); +export function createClient(opts: ICreateClientOpts | string): MatrixClient { + return new MatrixClient(amendClientOpts(opts)); +} + +export function createRoomWidgetClient( + widgetApi: WidgetApi, + capabilities: ICapabilities, + roomId: string, + opts: ICreateClientOpts | string, +): MatrixClient { + return new RoomWidgetClient(widgetApi, capabilities, roomId, amendClientOpts(opts)); } /** diff --git a/src/models/beacon.ts b/src/models/beacon.ts index 9df62bbe2b1..12db328b11f 100644 --- a/src/models/beacon.ts +++ b/src/models/beacon.ts @@ -17,7 +17,7 @@ limitations under the License. import { MBeaconEventContent } from "../@types/beacon"; import { M_TIMESTAMP } from "../@types/location"; import { BeaconInfoState, BeaconLocationState, parseBeaconContent, parseBeaconInfoContent } from "../content-helpers"; -import { MatrixEvent } from "../matrix"; +import { MatrixEvent } from "./event"; import { sortEventsByLatestContentTimestamp } from "../utils"; import { TypedEventEmitter } from "./typed-event-emitter"; diff --git a/src/models/thread.ts b/src/models/thread.ts index c451ccb8dc2..a68b88c0417 100644 --- a/src/models/thread.ts +++ b/src/models/thread.ts @@ -16,13 +16,14 @@ limitations under the License. import { Optional } from "matrix-events-sdk"; -import { MatrixClient, MatrixEventEvent, RelationType, RoomEvent } from "../matrix"; +import { MatrixClient } from "../client"; import { TypedReEmitter } from "../ReEmitter"; import { IRelationsRequestOpts } from "../@types/requests"; -import { IThreadBundledRelationship, MatrixEvent } from "./event"; +import { RelationType } from "../@types/event"; +import { IThreadBundledRelationship, MatrixEvent, MatrixEventEvent } from "./event"; import { Direction, EventTimeline } from "./event-timeline"; import { EventTimelineSet, EventTimelineSetHandlerMap } from './event-timeline-set'; -import { Room } from './room'; +import { Room, RoomEvent } from './room'; import { TypedEventEmitter } from "./typed-event-emitter"; import { RoomState } from "./room-state"; import { ServerControlledNamespacedValue } from "../NamespacedValue"; diff --git a/src/sliding-sync-sdk.ts b/src/sliding-sync-sdk.ts index 39fafec9056..e2dfcc381c0 100644 --- a/src/sliding-sync-sdk.ts +++ b/src/sliding-sync-sdk.ts @@ -35,7 +35,8 @@ import { SlidingSyncEvent, SlidingSyncState, } from "./sliding-sync"; -import { EventType, IPushRules } from "./matrix"; +import { EventType } from "./@types/event"; +import { IPushRules } from "./@types/PushRules"; import { PushProcessor } from "./pushprocessor"; // Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed @@ -290,7 +291,7 @@ export class SlidingSyncSdk { logger.debug("initial flag not set but no stored room exists for room ", roomId, roomData); return; } - room = createRoom(this.client, roomId, this.opts); + room = this.createRoom(roomId); } this.processRoomData(this.client, room, roomData); } @@ -388,6 +389,60 @@ export class SlidingSyncSdk { return this.syncStateData; } + // Helper functions which set up JS SDK structs are below and are identical to the sync v2 counterparts + + public createRoom(roomId: string): Room { // XXX cargoculted from sync.ts + const { timelineSupport } = this.client; + const room = new Room(roomId, this.client, this.client.getUserId(), { + lazyLoadMembers: this.opts.lazyLoadMembers, + pendingEventOrdering: this.opts.pendingEventOrdering, + timelineSupport, + }); + this.client.reEmitter.reEmit(room, [ + RoomEvent.Name, + RoomEvent.Redaction, + RoomEvent.RedactionCancelled, + RoomEvent.Receipt, + RoomEvent.Tags, + RoomEvent.LocalEchoUpdated, + RoomEvent.AccountData, + RoomEvent.MyMembership, + RoomEvent.Timeline, + RoomEvent.TimelineReset, + ]); + this.registerStateListeners(room); + return room; + } + + private registerStateListeners(room: Room): void { // XXX cargoculted from sync.ts + // we need to also re-emit room state and room member events, so hook it up + // to the client now. We need to add a listener for RoomState.members in + // order to hook them correctly. + this.client.reEmitter.reEmit(room.currentState, [ + RoomStateEvent.Events, + RoomStateEvent.Members, + RoomStateEvent.NewMember, + RoomStateEvent.Update, + ]); + room.currentState.on(RoomStateEvent.NewMember, (event, state, member) => { + member.user = this.client.getUser(member.userId); + this.client.reEmitter.reEmit(member, [ + RoomMemberEvent.Name, + RoomMemberEvent.Typing, + RoomMemberEvent.PowerLevel, + RoomMemberEvent.Membership, + ]); + }); + } + + /* + private deregisterStateListeners(room: Room): void { // XXX cargoculted from sync.ts + // could do with a better way of achieving this. + room.currentState.removeAllListeners(RoomStateEvent.Events); + room.currentState.removeAllListeners(RoomStateEvent.Members); + room.currentState.removeAllListeners(RoomStateEvent.NewMember); + } */ + private shouldAbortSync(error: MatrixError): boolean { if (error.errcode === "M_UNKNOWN_TOKEN") { // The logout already happened, we just need to stop. @@ -434,7 +489,7 @@ export class SlidingSyncSdk { if (roomData.invite_state) { const inviteStateEvents = mapEvents(this.client, room.roomId, roomData.invite_state); - this.processRoomEvents(room, inviteStateEvents); + this.injectRoomEvents(room, inviteStateEvents); if (roomData.initial) { room.recalculate(); this.client.store.storeRoom(room); @@ -504,11 +559,11 @@ export class SlidingSyncSdk { // reason to stop incrementally tracking notifications and // reset the timeline. this.client.resetNotifTimelineSet(); - registerStateListeners(this.client, room); + this.registerStateListeners(room); } } */ - this.processRoomEvents(room, stateEvents, timelineEvents, false); + this.injectRoomEvents(room, stateEvents, timelineEvents, false); // we deliberately don't add ephemeral events to the timeline room.addEphemeralEvents(ephemeralEvents); @@ -545,6 +600,7 @@ export class SlidingSyncSdk { } /** + * Injects events into a room's model. * @param {Room} room * @param {MatrixEvent[]} stateEventList A list of state events. This is the state * at the *START* of the timeline list if it is supplied. @@ -552,7 +608,7 @@ export class SlidingSyncSdk { * @param {boolean} fromCache whether the sync response came from cache * is earlier in time. Higher index is later. */ - private processRoomEvents( + public injectRoomEvents( room: Room, stateEventList: MatrixEvent[], timelineEventList?: MatrixEvent[], @@ -771,61 +827,6 @@ function ensureNameEvent(client: MatrixClient, roomId: string, roomData: MSC3575 return roomData; } -// Helper functions which set up JS SDK structs are below and are identical to the sync v2 counterparts, -// just outside the class. - -function createRoom(client: MatrixClient, roomId: string, opts: Partial): Room { // XXX cargoculted from sync.ts - const { timelineSupport } = client; - const room = new Room(roomId, client, client.getUserId(), { - lazyLoadMembers: opts.lazyLoadMembers, - pendingEventOrdering: opts.pendingEventOrdering, - timelineSupport, - }); - client.reEmitter.reEmit(room, [ - RoomEvent.Name, - RoomEvent.Redaction, - RoomEvent.RedactionCancelled, - RoomEvent.Receipt, - RoomEvent.Tags, - RoomEvent.LocalEchoUpdated, - RoomEvent.AccountData, - RoomEvent.MyMembership, - RoomEvent.Timeline, - RoomEvent.TimelineReset, - ]); - registerStateListeners(client, room); - return room; -} - -function registerStateListeners(client: MatrixClient, room: Room): void { // XXX cargoculted from sync.ts - // we need to also re-emit room state and room member events, so hook it up - // to the client now. We need to add a listener for RoomState.members in - // order to hook them correctly. - client.reEmitter.reEmit(room.currentState, [ - RoomStateEvent.Events, - RoomStateEvent.Members, - RoomStateEvent.NewMember, - RoomStateEvent.Update, - ]); - room.currentState.on(RoomStateEvent.NewMember, function(event, state, member) { - member.user = client.getUser(member.userId); - client.reEmitter.reEmit(member, [ - RoomMemberEvent.Name, - RoomMemberEvent.Typing, - RoomMemberEvent.PowerLevel, - RoomMemberEvent.Membership, - ]); - }); -} - -/* -function deregisterStateListeners(room: Room): void { // XXX cargoculted from sync.ts - // could do with a better way of achieving this. - room.currentState.removeAllListeners(RoomStateEvent.Events); - room.currentState.removeAllListeners(RoomStateEvent.Members); - room.currentState.removeAllListeners(RoomStateEvent.NewMember); -} */ - function mapEvents(client: MatrixClient, roomId: string, events: object[], decrypt = true): MatrixEvent[] { const mapper = client.getEventMapper({ decrypt }); return (events as Array).map(function(e) { diff --git a/src/sync.ts b/src/sync.ts index 4abd4fb5bb2..3e124082130 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -398,7 +398,7 @@ export class SyncApi { // events so that clients can start back-paginating. room.getLiveTimeline().setPaginationToken(leaveObj.timeline.prev_batch, EventTimeline.BACKWARDS); - await this.processRoomEvents(room, stateEvents, events); + await this.injectRoomEvents(room, stateEvents, events); room.recalculate(); client.store.storeRoom(room); @@ -430,7 +430,7 @@ export class SyncApi { response.messages.chunk = response.messages.chunk || []; response.state = response.state || []; - // FIXME: Mostly duplicated from processRoomEvents but not entirely + // FIXME: Mostly duplicated from injectRoomEvents but not entirely // because "state" in this API is at the BEGINNING of the chunk const oldStateEvents = utils.deepCopy(response.state) .map(client.getEventMapper()); @@ -1252,7 +1252,7 @@ export class SyncApi { const room = inviteObj.room; const stateEvents = this.mapSyncEventsFormat(inviteObj.invite_state, room); - await this.processRoomEvents(room, stateEvents); + await this.injectRoomEvents(room, stateEvents); if (inviteObj.isBrandNewRoom) { room.recalculate(); client.store.storeRoom(room); @@ -1358,7 +1358,7 @@ export class SyncApi { } } - await this.processRoomEvents(room, stateEvents, events, syncEventData.fromCache); + await this.injectRoomEvents(room, stateEvents, events, syncEventData.fromCache); // set summary after processing events, // because it will trigger a name calculation @@ -1410,7 +1410,7 @@ export class SyncApi { const events = this.mapSyncEventsFormat(leaveObj.timeline, room); const accountDataEvents = this.mapSyncEventsFormat(leaveObj.account_data); - await this.processRoomEvents(room, stateEvents, events); + await this.injectRoomEvents(room, stateEvents, events); room.addAccountData(accountDataEvents); room.recalculate(); @@ -1649,14 +1649,15 @@ export class SyncApi { } /** + * Injects events into a room's model. * @param {Room} room * @param {MatrixEvent[]} stateEventList A list of state events. This is the state * at the *START* of the timeline list if it is supplied. * @param {MatrixEvent[]} [timelineEventList] A list of timeline events, including threaded. Lower index - * @param {boolean} fromCache whether the sync response came from cache * is earlier in time. Higher index is later. + * @param {boolean} fromCache whether the sync response came from cache */ - private async processRoomEvents( + public async injectRoomEvents( room: Room, stateEventList: MatrixEvent[], timelineEventList?: MatrixEvent[], diff --git a/src/utils.ts b/src/utils.ts index 6cf459097c1..18d6c29a0fb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -24,7 +24,7 @@ import unhomoglyph from "unhomoglyph"; import promiseRetry from "p-retry"; import type * as NodeCrypto from "crypto"; -import { MatrixEvent } from "."; +import { MatrixEvent } from "./models/event"; import { M_TIMESTAMP } from "./@types/location"; /** diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index e1932cdf9c6..9bd2473ae14 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -557,6 +557,13 @@ export class MatrixCall extends TypedEventEmitter candidate.toJSON()) }; logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { await this.sendVoipEvent(EventType.CallCandidates, content); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 1d130739910..7c926be6b7a 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -269,6 +269,8 @@ export class CallEventHandler { } catch (e) { if (e.code === GroupCallErrorCode.UnknownDevice) { groupCall?.emit(GroupCallEvent.Error, e); + } else { + logger.error(e); } } this.calls.set(call.callId, call); diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index b30f0dd8a1a..9d7f2e5afd4 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -16,7 +16,7 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { RoomStateEvent } from '../models/room-state'; -import { MatrixClient } from '../client'; +import { MatrixClient, ClientEvent } from '../client'; import { GroupCall, GroupCallIntent, @@ -25,9 +25,9 @@ import { } from "./groupCall"; import { Room } from "../models/room"; import { RoomState } from "../models/room-state"; +import { RoomMember } from "../models/room-member"; import { logger } from '../logger'; import { EventType } from "../@types/event"; -import { ClientEvent, RoomMember } from '../matrix'; export enum GroupCallEventHandlerEvent { Incoming = "GroupCall.incoming", diff --git a/yarn.lock b/yarn.lock index 9476b4d3406..09cc30f64de 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1455,6 +1455,11 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@tootallnate/once@2": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" + integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== + "@types/babel-types@*", "@types/babel-types@^7.0.0": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/babel-types/-/babel-types-7.0.11.tgz#263b113fa396fac4373188d73225297fb86f19a9" @@ -1517,6 +1522,11 @@ resolved "https://registry.yarnpkg.com/@types/content-type/-/content-type-1.1.5.tgz#aa02dca40864749a9e2bf0161a6216da57e3ede5" integrity sha512-dgMN+syt1xb7Hk8LU6AODOfPlvz5z1CbXpPuJE5ZrX9STfBOIXF09pEB8N7a97WT9dbngt3ksDCm6GW6yMrxfQ== +"@types/events@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== + "@types/graceful-fs@^4.1.3": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" @@ -1551,6 +1561,15 @@ jest-matcher-utils "^28.0.0" pretty-format "^28.0.0" +"@types/jsdom@^16.2.4": + version "16.2.15" + resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.15.tgz#6c09990ec43b054e49636cba4d11d54367fc90d6" + integrity sha512-nwF87yjBKuX/roqGYerZZM0Nv1pZDMAT5YhOHYeM/72Fic+VEqJh4nyoqoapzJnW3pUlfxPY5FhgsJtM+dRnQQ== + dependencies: + "@types/node" "*" + "@types/parse5" "^6.0.3" + "@types/tough-cookie" "*" + "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" @@ -1589,6 +1608,11 @@ resolved "https://registry.yarnpkg.com/@types/node/-/node-16.11.45.tgz#155b13a33c665ef2b136f7f245fa525da419e810" integrity sha512-3rKg/L5x0rofKuuUt5zlXzOnKyIHXmIu5R8A0TuNDMF2062/AOIDBciFIjToLEJ/9F9DzkHNot+BpNsMI1OLdQ== +"@types/parse5@^6.0.3": + version "6.0.3" + resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb" + integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g== + "@types/prettier@^2.1.5": version "2.6.4" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.6.4.tgz#ad899dad022bab6b5a9f0a0fe67c2f7a4a8950ed" @@ -1724,6 +1748,11 @@ JSONStream@^1.0.3: jsonparse "^1.2.0" through ">=2.2.7 <3" +abab@^2.0.5, abab@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291" + integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== + ace-builds@^1.4.13: version "1.8.1" resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.8.1.tgz#5d318fa13d7e6ea947f8a50e42c570c573b29529" @@ -1736,6 +1765,14 @@ acorn-globals@^3.0.0: dependencies: acorn "^4.0.4" +acorn-globals@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" + integrity sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg== + dependencies: + acorn "^7.1.1" + acorn-walk "^7.1.1" + acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" @@ -1750,7 +1787,7 @@ acorn-node@^1.2.0, acorn-node@^1.3.0, acorn-node@^1.5.2, acorn-node@^1.8.2: acorn-walk "^7.0.0" xtend "^4.0.2" -acorn-walk@^7.0.0: +acorn-walk@^7.0.0, acorn-walk@^7.1.1: version "7.2.0" resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== @@ -1765,7 +1802,7 @@ acorn@^4.0.4, acorn@~4.0.2: resolved "https://registry.yarnpkg.com/acorn/-/acorn-4.0.13.tgz#105495ae5361d697bd195c825192e1ad7f253787" integrity sha512-fu2ygVGuMmlzG8ZeRJ0bvR41nsAkxxhbyk8bZ1SS521Z7vmgJFTQQlfz/Mp/nJexGBz+v8sC9bM6+lNgskt4Ug== -acorn@^7.0.0: +acorn@^7.0.0, acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== @@ -1775,6 +1812,13 @@ acorn@^8.5.0, acorn@^8.7.1: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + ajv@^6.10.0, ajv@^6.12.3, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -2191,6 +2235,11 @@ browser-pack@^6.0.1: through2 "^2.0.0" umd "^3.0.0" +browser-process-hrtime@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" + integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== + browser-request@^0.3.3: version "0.3.3" resolved "https://registry.yarnpkg.com/browser-request/-/browser-request-0.3.3.tgz#9ece5b5aca89a29932242e18bf933def9876cc17" @@ -2592,7 +2641,7 @@ combine-source-map@^0.8.0, combine-source-map@~0.8.0: lodash.memoize "~3.0.3" source-map "~0.5.3" -combined-stream@^1.0.6, combined-stream@~1.0.6: +combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -2746,6 +2795,23 @@ crypto-browserify@^3.0.0: randombytes "^2.0.0" randomfill "^1.0.3" +cssom@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" + integrity sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw== + +cssom@~0.3.6: + version "0.3.8" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" + integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== + +cssstyle@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" + integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== + dependencies: + cssom "~0.3.6" + d@1, d@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a" @@ -2766,11 +2832,27 @@ dashdash@^1.12.0: dependencies: assert-plus "^1.0.0" +data-urls@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-3.0.2.tgz#9cf24a477ae22bcef5cd5f6f0bfbc1d2d3be9143" + integrity sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ== + dependencies: + abab "^2.0.6" + whatwg-mimetype "^3.0.0" + whatwg-url "^11.0.0" + de-indent@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d" integrity sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg== +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -2785,24 +2867,22 @@ debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - decamelize@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== +decimal.js@^10.3.1: + version "10.3.1" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" + integrity sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ== + dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA== -deep-is@^0.1.3: +deep-is@^0.1.3, deep-is@~0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== @@ -2929,6 +3009,13 @@ domexception@^1.0.1: dependencies: webidl-conversions "^4.0.2" +domexception@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/domexception/-/domexception-4.0.0.tgz#4ad1be56ccadc86fc76d033353999a8037d03673" + integrity sha512-A2is4PLG+eeSfoTMA95/s4pvAoSo2mKtiM5jlHkAVewmiO8ISFTFKZjH7UAM1Atli/OT/7JHOrJRJiMKUZKYBw== + dependencies: + webidl-conversions "^7.0.0" + duplexer2@^0.1.2, duplexer2@~0.1.0, duplexer2@~0.1.2: version "0.1.4" resolved "https://registry.yarnpkg.com/duplexer2/-/duplexer2-0.1.4.tgz#8b12dab878c0d69e3e7891051662a32fc6bddcc1" @@ -3085,6 +3172,18 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== +escodegen@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" + integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw== + dependencies: + esprima "^4.0.1" + estraverse "^5.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.6.1" + eslint-config-google@^0.14.0: version "0.14.0" resolved "https://registry.yarnpkg.com/eslint-config-google/-/eslint-config-google-0.14.0.tgz#4f5f8759ba6e11b424294a219dbfa18c508bcc1a" @@ -3218,7 +3317,7 @@ espree@^9.3.2: acorn-jsx "^5.3.2" eslint-visitor-keys "^3.3.0" -esprima@^4.0.0, esprima@~4.0.0: +esprima@^4.0.0, esprima@^4.0.1, esprima@~4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== @@ -3269,7 +3368,7 @@ event-emitter@^0.3.5: d "1" es5-ext "~0.10.14" -events@^3.0.0: +events@^3.0.0, events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== @@ -3373,7 +3472,7 @@ fast-json-stable-stringify@^2.0.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-levenshtein@^2.0.6: +fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== @@ -3492,6 +3591,15 @@ form-data@^2.5.0: combined-stream "^1.0.6" mime-types "^2.1.12" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" @@ -3735,6 +3843,13 @@ hmac-drbg@^1.0.1: minimalistic-assert "^1.0.0" minimalistic-crypto-utils "^1.0.1" +html-encoding-sniffer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz#2cb1a8cf0db52414776e5b2a7a04d5dd98158de9" + integrity sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA== + dependencies: + whatwg-encoding "^2.0.0" + html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" @@ -3745,6 +3860,15 @@ htmlescape@^1.1.0: resolved "https://registry.yarnpkg.com/htmlescape/-/htmlescape-1.1.1.tgz#3a03edc2214bca3b66424a3e7959349509cb0351" integrity sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg== +http-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" + integrity sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w== + dependencies: + "@tootallnate/once" "2" + agent-base "6" + debug "4" + http-signature@~1.2.0: version "1.2.0" resolved "https://registry.yarnpkg.com/http-signature/-/http-signature-1.2.0.tgz#9aecd925114772f3d95b65a60abb8f7c18fbace1" @@ -3759,11 +3883,26 @@ https-browserify@^1.0.0: resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg== +https-proxy-agent@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" + integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== + dependencies: + agent-base "6" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +iconv-lite@0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + ieee754@^1.1.4: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -3970,6 +4109,11 @@ is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== +is-potential-custom-element-name@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" + integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ== + is-promise@^2.0.0, is-promise@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" @@ -4206,6 +4350,20 @@ jest-each@^28.1.3: jest-util "^28.1.3" pretty-format "^28.1.3" +jest-environment-jsdom@^28.1.3: + version "28.1.3" + resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-28.1.3.tgz#2d4e5d61b7f1d94c3bddfbb21f0308ee506c09fb" + integrity sha512-HnlGUmZRdxfCByd3GM2F100DgQOajUBzEitjGqIREcb45kGjZvRrKUdlaF6escXBdcXNl0OBh+1ZrfeZT3GnAg== + dependencies: + "@jest/environment" "^28.1.3" + "@jest/fake-timers" "^28.1.3" + "@jest/types" "^28.1.3" + "@types/jsdom" "^16.2.4" + "@types/node" "*" + jest-mock "^28.1.3" + jest-util "^28.1.3" + jsdom "^19.0.0" + jest-environment-node@^28.1.3: version "28.1.3" resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-28.1.3.tgz#7e74fe40eb645b9d56c0c4b70ca4357faa349be5" @@ -4527,6 +4685,39 @@ jsdoc@^3.6.6: taffydb "2.6.2" underscore "~1.13.2" +jsdom@^19.0.0: + version "19.0.0" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-19.0.0.tgz#93e67c149fe26816d38a849ea30ac93677e16b6a" + integrity sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A== + dependencies: + abab "^2.0.5" + acorn "^8.5.0" + acorn-globals "^6.0.0" + cssom "^0.5.0" + cssstyle "^2.3.0" + data-urls "^3.0.1" + decimal.js "^10.3.1" + domexception "^4.0.0" + escodegen "^2.0.0" + form-data "^4.0.0" + html-encoding-sniffer "^3.0.0" + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.0" + is-potential-custom-element-name "^1.0.1" + nwsapi "^2.2.0" + parse5 "6.0.1" + saxes "^5.0.1" + symbol-tree "^3.2.4" + tough-cookie "^4.0.0" + w3c-hr-time "^1.0.2" + w3c-xmlserializer "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-encoding "^2.0.0" + whatwg-mimetype "^3.0.0" + whatwg-url "^10.0.0" + ws "^8.2.3" + xml-name-validator "^4.0.0" + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -4647,6 +4838,14 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" +levn@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" + integrity sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA== + dependencies: + prelude-ls "~1.1.2" + type-check "~0.3.2" + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -4813,6 +5012,14 @@ matrix-mock-request@^2.1.2: dependencies: expect "^28.1.0" +matrix-widget-api@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.0.0.tgz#0cde6839cca66ad817ab12aca3490ccc8bac97d1" + integrity sha512-cy8p/8EteRPTFIAw7Q9EgPUJc2jD19ZahMR8bMKf2NkILDcjuPMC0UWnsJyB3fSnlGw+VbGepttRpULM31zX8Q== + dependencies: + "@types/events" "^3.0.0" + events "^3.2.0" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" @@ -5016,6 +5223,11 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +nwsapi@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.1.tgz#10a9f268fbf4c461249ebcfe38e359aa36e2577c" + integrity sha512-JYOWTeFoS0Z93587vRJgASD5Ut11fYl5NyihP3KrYBvMe1FRRs6RN7m20SA/16GM4P6hTnZjT+UmDOt38UeXNg== + oauth-sign@~0.9.0: version "0.9.0" resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" @@ -5069,6 +5281,18 @@ onetime@^5.1.2: dependencies: mimic-fn "^2.1.0" +optionator@^0.8.1: + version "0.8.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" + integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== + dependencies: + deep-is "~0.1.3" + fast-levenshtein "~2.0.6" + levn "~0.3.0" + prelude-ls "~1.1.2" + type-check "~0.3.2" + word-wrap "~1.2.3" + optionator@^0.9.1: version "0.9.1" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" @@ -5200,6 +5424,11 @@ parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" +parse5@6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" + integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== + path-browserify@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/path-browserify/-/path-browserify-1.0.1.tgz#d98454a9c3753d5790860f16f68867b9e46be1fd" @@ -5295,6 +5524,11 @@ prelude-ls@^1.2.1: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== +prelude-ls@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" + integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== + pretty-format@^28.0.0, pretty-format@^28.1.3: version "28.1.3" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-28.1.3.tgz#c9fba8cedf99ce50963a11b27d982a9ae90970d5" @@ -5349,7 +5583,7 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== -psl@^1.1.28: +psl@^1.1.28, psl@^1.1.33: version "1.9.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== @@ -5815,11 +6049,18 @@ safe-buffer@~5.1.0, safe-buffer@~5.1.1: resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" integrity sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g== -safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: +"safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.0.2, safer-buffer@^2.1.0, safer-buffer@~2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +saxes@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" + integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== + dependencies: + xmlchars "^2.2.0" + sdp-transform@^2.14.1: version "2.14.1" resolved "https://registry.yarnpkg.com/sdp-transform/-/sdp-transform-2.14.1.tgz#2bb443583d478dee217df4caa284c46b870d5827" @@ -6143,6 +6384,11 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +symbol-tree@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" + integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== + syntax-error@^1.1.1: version "1.4.0" resolved "https://registry.yarnpkg.com/syntax-error/-/syntax-error-1.4.0.tgz#2d9d4ff5c064acb711594a3e3b95054ad51d907c" @@ -6247,6 +6493,15 @@ token-stream@0.0.1: resolved "https://registry.yarnpkg.com/token-stream/-/token-stream-0.0.1.tgz#ceeefc717a76c4316f126d0b9dbaa55d7e7df01a" integrity sha512-nfjOAu/zAWmX9tgwi5NRp7O7zTDUD1miHiB40klUnAh9qnL1iXdgzcz/i5dMaL5jahcBAaSfmNOBBJBLJW8TEg== +tough-cookie@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" + integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.1.2" + tough-cookie@~2.5.0: version "2.5.0" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.5.0.tgz#cd9fb2a0aa1d5a12b473bd9fb96fa3dcff65ade2" @@ -6262,6 +6517,13 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -6350,6 +6612,13 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" +type-check@~0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" + integrity sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg== + dependencies: + prelude-ls "~1.1.2" + type-detect@4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" @@ -6488,6 +6757,11 @@ universal-user-agent@^6.0.0: resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== +universalify@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" + integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== + update-browserslist-db@^1.0.4: version "1.0.5" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.5.tgz#be06a5eedd62f107b7c19eb5bcefb194411abf38" @@ -6604,6 +6878,20 @@ vue2-ace-editor@^0.0.15: dependencies: brace "^0.11.0" +w3c-hr-time@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" + integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== + dependencies: + browser-process-hrtime "^1.0.0" + +w3c-xmlserializer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz#06cdc3eefb7e4d0b20a560a5a3aeb0d2d9a65923" + integrity sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg== + dependencies: + xml-name-validator "^4.0.0" + walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" @@ -6626,6 +6914,39 @@ webidl-conversions@^6.1.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-6.1.0.tgz#9111b4d7ea80acd40f5270d666621afa78b69514" integrity sha512-qBIvFLGiBpLjfwmYAaHPXsn+ho5xZnGvyGvsarywGNc8VyQJUMHJ8OBKGGrPER0okBeMDaan4mNBlgBROxuI8w== +webidl-conversions@^7.0.0: + version "7.0.0" + resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" + integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== + +whatwg-encoding@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53" + integrity sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg== + dependencies: + iconv-lite "0.6.3" + +whatwg-mimetype@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" + integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q== + +whatwg-url@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-10.0.0.tgz#37264f720b575b4a311bd4094ed8c760caaa05da" + integrity sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" @@ -6686,7 +7007,7 @@ with@^5.0.0: acorn "^3.1.0" acorn-globals "^3.0.0" -word-wrap@^1.2.3: +word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== @@ -6718,11 +7039,26 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" +ws@^8.2.3: + version "8.8.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0" + integrity sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA== + +xml-name-validator@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835" + integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== + xml@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw== +xmlchars@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" + integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== + xmlcreate@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/xmlcreate/-/xmlcreate-2.0.4.tgz#0c5ab0f99cdd02a81065fa9cd8f8ae87624889be" From 020743141b050425fc2cf5a2c3c69d701b6f6cc1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 10 Aug 2022 18:07:49 +0100 Subject: [PATCH 224/291] Tidy up imports (#2584) Duplicate 'call' imports --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 7c3a891a6a4..83239fadeae 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -8,13 +8,13 @@ import { CallErrorCode, genCallID, MatrixCall, setTracksEnabled, + createNewMatrixCall, } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; import { logger } from "../logger"; import { ReEmitter } from "../ReEmitter"; import { SDPStreamMetadataPurpose } from "./callEventTypes"; -import { createNewMatrixCall } from "./call"; import { ISendEventResponse } from "../@types/requests"; import { MatrixEvent } from "../models/event"; import { EventType } from "../@types/event"; From e8f682f452e1d67226abeb8fbaca4faac2c54747 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 16 Aug 2022 18:22:36 +0100 Subject: [PATCH 225/291] Test placing a call in a group call (#2593) * Test placing a call in a group call Refactors a bit of the call testing stuff Fixes https://github.com/vector-im/element-call/issues/521 * Unused imports * Use expect.toHaveBeenCalledWith() * Types * More types * Add comment on mock typing * Use toHaveBeenCalledWith() * Initialise groupcall & room in beforeEach * Initialise mockMediahandler sensibly * Add type params to mock * Rename mute tests * Move comment * Join / leave in parallel * Remove leftover expect --- spec/test-utils/webrtc.ts | 73 +++++- spec/unit/webrtc/call.spec.ts | 28 +-- spec/unit/webrtc/groupCall.spec.ts | 353 +++++++++++++++++++++++------ 3 files changed, 360 insertions(+), 94 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index ed124ba7a14..d7968caa244 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -70,20 +70,41 @@ export class MockAudioContext { } export class MockRTCPeerConnection { + private static instances: MockRTCPeerConnection[] = []; + + private negotiationNeededListener: () => void; + private needsNegotiation = false; localDescription: RTCSessionDescription; + public static triggerAllNegotiations() { + for (const inst of this.instances) { + inst.doNegotiation(); + } + } + + public static resetInstances() { + this.instances = []; + } + constructor() { this.localDescription = { sdp: DUMMY_SDP, type: 'offer', toJSON: function() { }, }; + + MockRTCPeerConnection.instances.push(this); } - addEventListener() { } + addEventListener(type: string, listener: () => void) { + if (type === 'negotiationneeded') this.negotiationNeededListener = listener; + } createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } createOffer() { - return Promise.resolve({}); + return Promise.resolve({ + type: 'offer', + sdp: DUMMY_SDP, + }); } setRemoteDescription() { return Promise.resolve(); @@ -93,7 +114,17 @@ export class MockRTCPeerConnection { } close() { } getStats() { return []; } - addTrack(track: MockMediaStreamTrack) { return new MockRTCRtpSender(track); } + addTrack(track: MockMediaStreamTrack) { + this.needsNegotiation = true; + return new MockRTCRtpSender(track); + } + + doNegotiation() { + if (this.needsNegotiation && this.negotiationNeededListener) { + this.needsNegotiation = false; + this.negotiationNeededListener(); + } + } } export class MockRTCRtpSender { @@ -140,6 +171,10 @@ export class MockMediaStream { this.dispatchEvent("addtrack"); } removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } + + clone() { + return new MockMediaStream(this.id, this.tracks); + } } export class MockMediaDeviceInfo { @@ -149,6 +184,9 @@ export class MockMediaDeviceInfo { } export class MockMediaHandler { + public userMediaStreams: MediaStream[] = []; + public screensharingStreams: MediaStream[] = []; + getUserMediaStream(audio: boolean, video: boolean) { const tracks = []; if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); @@ -160,3 +198,32 @@ export class MockMediaHandler { hasAudioDevice() { return true; } stopAllStreams() {} } + +export function installWebRTCMocks() { + global.navigator = { + mediaDevices: { + // @ts-ignore Mock + getUserMedia: () => new MockMediaStream("local_stream"), + // @ts-ignore Mock + enumerateDevices: async () => [new MockMediaDeviceInfo("audio"), new MockMediaDeviceInfo("video")], + }, + }; + + global.window = { + // @ts-ignore Mock + RTCPeerConnection: MockRTCPeerConnection, + // @ts-ignore Mock + RTCSessionDescription: {}, + // @ts-ignore Mock + RTCIceCandidate: {}, + getUserMedia: () => new MockMediaStream("local_stream"), + }; + // @ts-ignore Mock + global.document = {}; + + // @ts-ignore Mock + global.AudioContext = MockAudioContext; + + // @ts-ignore Mock + global.RTCRtpReceiver = {}; +} diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 9fa33e3ece5..4f0e177d79a 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -22,9 +22,7 @@ import { MockMediaHandler, MockMediaStream, MockMediaStreamTrack, - MockMediaDeviceInfo, - MockRTCPeerConnection, - MockAudioContext, + installWebRTCMocks, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; @@ -48,29 +46,7 @@ describe('Call', function() { prevDocument = global.document; prevWindow = global.window; - global.navigator = { - mediaDevices: { - // @ts-ignore Mock - getUserMedia: () => new MockMediaStream("local_stream"), - // @ts-ignore Mock - enumerateDevices: async () => [new MockMediaDeviceInfo("audio"), new MockMediaDeviceInfo("video")], - }, - }; - - global.window = { - // @ts-ignore Mock - RTCPeerConnection: MockRTCPeerConnection, - // @ts-ignore Mock - RTCSessionDescription: {}, - // @ts-ignore Mock - RTCIceCandidate: {}, - getUserMedia: () => new MockMediaStream("local_stream"), - }; - // @ts-ignore Mock - global.document = {}; - - // @ts-ignore Mock - global.AudioContext = MockAudioContext; + installWebRTCMocks(); client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); // We just stub out sendEvent: we're not interested in testing the client's diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 36669ea151e..4bba4a91df7 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -14,85 +14,308 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { EventType, GroupCallIntent, GroupCallType, Room, RoomMember } from '../../../src'; +import { EventType, GroupCallIntent, GroupCallType, MatrixEvent, Room, RoomMember } from '../../../src'; import { GroupCall } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; -import { MockAudioContext, MockMediaHandler } from '../../test-utils/webrtc'; +import { installWebRTCMocks, MockMediaHandler, MockRTCPeerConnection } from '../../test-utils/webrtc'; +import { ReEmitter } from '../../../src/ReEmitter'; +import { TypedEventEmitter } from '../../../src/models/typed-event-emitter'; +import { MediaHandler } from '../../../src/webrtc/mediaHandler'; -const FAKE_SELF_USER_ID = "@me:test.dummy"; -const FAKE_SELF_DEVICE_ID = "AAAAAA"; -const FAKE_SELF_SESSION_ID = "1"; const FAKE_ROOM_ID = "!fake:test.dummy"; +const FAKE_CONF_ID = "fakegroupcallid"; + +const FAKE_USER_ID_1 = "@alice:test.dummy"; +const FAKE_DEVICE_ID_1 = "@AAAAAA"; +const FAKE_SESSION_ID_1 = "alice1"; +const FAKE_USER_ID_2 = "@bob:test.dummy"; +const FAKE_DEVICE_ID_2 = "@BBBBBB"; +const FAKE_SESSION_ID_2 = "bob1"; + +class MockCallMatrixClient { + public mediaHandler: MediaHandler = new MockMediaHandler() as unknown as MediaHandler; + + constructor(public userId: string, public deviceId: string, public sessionId: string) { + } + + groupCallEventHandler = { + groupCalls: new Map(), + }; + + callEventHandler = { + calls: new Map(), + }; + + sendStateEvent = jest.fn(); + + getMediaHandler() { return this.mediaHandler; } + + getUserId() { return this.userId; } + + getDeviceId() { return this.deviceId; } + getSessionId() { return this.sessionId; } + + emit = jest.fn(); + on = jest.fn(); + removeListener = jest.fn(); + getTurnServers = () => []; + isFallbackICEServerAllowed = () => false; + reEmitter = new ReEmitter(new TypedEventEmitter()); + getUseE2eForGroupCall = () => false; + checkTurnServers = () => null; +} describe('Group Call', function() { beforeEach(function() { - // @ts-ignore Mock - global.AudioContext = MockAudioContext; + installWebRTCMocks(); }); - it("sends state event to room when creating", async () => { - const mockSendState = jest.fn(); + describe('Basic functionality', function() { + let mockSendState: jest.Mock; + let mockClient: MatrixClient; + let room: Room; + let groupCall: GroupCall; + + beforeEach(function() { + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockSendState = typedMockClient.sendStateEvent; + + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); + }); + + it("sends state event to room when creating", async () => { + await groupCall.create(); + + expect(mockSendState).toHaveBeenCalledWith( + FAKE_ROOM_ID, EventType.GroupCallPrefix, expect.objectContaining({ + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + }), + groupCall.groupCallId, + ); + }); + + it("sends member state event to room on enter", async () => { + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + + await groupCall.create(); + + try { + await groupCall.enter(); + + expect(mockSendState).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.GroupCallMemberPrefix, + expect.objectContaining({ + "m.calls": [ + expect.objectContaining({ + "m.call_id": groupCall.groupCallId, + "m.devices": [ + expect.objectContaining({ + device_id: FAKE_DEVICE_ID_1, + }), + ], + }), + ], + }), + FAKE_USER_ID_1, + ); + } finally { + groupCall.leave(); + } + }); + + it("starts with mic unmuted in regular calls", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + expect(groupCall.isMicrophoneMuted()).toEqual(false); + } finally { + groupCall.leave(); + } + }); + + it("starts with mic muted in PTT calls", async () => { + try { + // replace groupcall with a PTT one for this test + // we will probably want a dedicated test suite for PTT calls, so when we do, + // this can go in there instead. + groupCall = new GroupCall(mockClient, room, GroupCallType.Video, true, GroupCallIntent.Prompt); + + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + expect(groupCall.isMicrophoneMuted()).toEqual(true); + } finally { + groupCall.leave(); + } + }); + + it("disables audio stream when audio is set to muted", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + await groupCall.setMicrophoneMuted(true); - const mockClient = { - sendStateEvent: mockSendState, - groupCallEventHandler: { - groupCalls: new Map(), - }, - } as unknown as MatrixClient; + expect(groupCall.isMicrophoneMuted()).toEqual(true); + } finally { + groupCall.leave(); + } + }); - const room = new Room(FAKE_ROOM_ID, mockClient, FAKE_SELF_USER_ID); - const groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); + it("starts with video unmuted in regular calls", async () => { + try { + await groupCall.create(); - await groupCall.create(); + await groupCall.initLocalCallFeed(); - expect(mockSendState.mock.calls[0][0]).toEqual(FAKE_ROOM_ID); - expect(mockSendState.mock.calls[0][1]).toEqual(EventType.GroupCallPrefix); - expect(mockSendState.mock.calls[0][2]["m.type"]).toEqual(GroupCallType.Video); - expect(mockSendState.mock.calls[0][2]["m.intent"]).toEqual(GroupCallIntent.Prompt); + expect(groupCall.isLocalVideoMuted()).toEqual(false); + } finally { + groupCall.leave(); + } + }); + + it("disables video stream when video is set to muted", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + await groupCall.setLocalVideoMuted(true); + + expect(groupCall.isLocalVideoMuted()).toEqual(true); + } finally { + groupCall.leave(); + } + }); }); - it("sends member state event to room on enter", async () => { - const mockSendState = jest.fn(); - const mockMediaHandler = new MockMediaHandler(); - - const mockClient = { - sendStateEvent: mockSendState, - groupCallEventHandler: { - groupCalls: new Map(), - }, - callEventHandler: { - calls: new Map(), - }, - mediaHandler: mockMediaHandler, - getMediaHandler: () => mockMediaHandler, - getUserId: () => FAKE_SELF_USER_ID, - getDeviceId: () => FAKE_SELF_DEVICE_ID, - getSessionId: () => FAKE_SELF_SESSION_ID, - emit: jest.fn(), - on: jest.fn(), - removeListener: jest.fn(), - } as unknown as MatrixClient; - - const room = new Room(FAKE_ROOM_ID, mockClient, FAKE_SELF_USER_ID); - const groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); - - room.currentState.members[FAKE_SELF_USER_ID] = { - userId: FAKE_SELF_USER_ID, - } as unknown as RoomMember; - - await groupCall.create(); - - try { - await groupCall.enter(); - - expect(mockSendState.mock.lastCall[0]).toEqual(FAKE_ROOM_ID); - expect(mockSendState.mock.lastCall[1]).toEqual(EventType.GroupCallMemberPrefix); - expect(mockSendState.mock.lastCall[2]['m.calls'].length).toEqual(1); - expect(mockSendState.mock.lastCall[2]['m.calls'][0]["m.call_id"]).toEqual(groupCall.groupCallId); - expect(mockSendState.mock.lastCall[2]['m.calls'][0]['m.devices'].length).toEqual(1); - expect(mockSendState.mock.lastCall[2]['m.calls'][0]['m.devices'][0].device_id).toEqual(FAKE_SELF_DEVICE_ID); - } finally { - groupCall.leave(); - } + describe('Placing calls', function() { + let groupCall1: GroupCall; + let groupCall2: GroupCall; + let client1: MatrixClient; + let client2: MatrixClient; + + beforeEach(function() { + MockRTCPeerConnection.resetInstances(); + + client1 = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ) as unknown as MatrixClient; + + client2 = new MockCallMatrixClient( + FAKE_USER_ID_2, FAKE_DEVICE_ID_2, FAKE_SESSION_ID_2, + ) as unknown as MatrixClient; + + client1.sendStateEvent = client2.sendStateEvent = (roomId, eventType, content, statekey) => { + if (eventType === EventType.GroupCallMemberPrefix) { + const fakeEvent = { + getContent: () => content, + getRoomId: () => FAKE_ROOM_ID, + getStateKey: () => statekey, + } as unknown as MatrixEvent; + + let subMap = client1Room.currentState.events.get(eventType); + if (!subMap) { + subMap = new Map(); + client1Room.currentState.events.set(eventType, subMap); + client2Room.currentState.events.set(eventType, subMap); + } + // since we cheat & use the same maps for each, we can + // just add it once. + subMap.set(statekey, fakeEvent); + + groupCall1.onMemberStateChanged(fakeEvent); + groupCall2.onMemberStateChanged(fakeEvent); + } + return Promise.resolve(null); + }; + + const client1Room = new Room(FAKE_ROOM_ID, client1, FAKE_USER_ID_1); + + const client2Room = new Room(FAKE_ROOM_ID, client2, FAKE_USER_ID_2); + + groupCall1 = new GroupCall( + client1, client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, + ); + + groupCall2 = new GroupCall( + client2, client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, + ); + + client1Room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + client1Room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + } as unknown as RoomMember; + + client2Room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + client2Room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + } as unknown as RoomMember; + }); + + afterEach(function() { + MockRTCPeerConnection.resetInstances(); + }); + + it("Places a call to a peer", async function() { + await groupCall1.create(); + + try { + // keep this as its own variable so we have it typed as a mock + // rather than its type in the client object + const mockSendToDevice = jest.fn, [ + eventType: string, + contentMap: { [userId: string]: { [deviceId: string]: Record } }, + txnId?: string, + ]>(); + + const toDeviceProm = new Promise(resolve => { + mockSendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + client1.sendToDevice = mockSendToDevice; + + await Promise.all([groupCall1.enter(), groupCall2.enter()]); + + MockRTCPeerConnection.triggerAllNegotiations(); + + await toDeviceProm; + + expect(mockSendToDevice.mock.calls[0][0]).toBe("m.call.invite"); + + const toDeviceCallContent = mockSendToDevice.mock.calls[0][1]; + expect(Object.keys(toDeviceCallContent).length).toBe(1); + expect(Object.keys(toDeviceCallContent)[0]).toBe(FAKE_USER_ID_2); + + const toDeviceBobDevices = toDeviceCallContent[FAKE_USER_ID_2]; + expect(Object.keys(toDeviceBobDevices).length).toBe(1); + expect(Object.keys(toDeviceBobDevices)[0]).toBe(FAKE_DEVICE_ID_2); + + const bobDeviceMessage = toDeviceBobDevices[FAKE_DEVICE_ID_2]; + expect(bobDeviceMessage.conf_id).toBe(FAKE_CONF_ID); + } finally { + await Promise.all([groupCall1.leave(), groupCall2.leave()]); + } + }); }); }); From c698317f3f18c51deccb465f87e36961a6f27b66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Wed, 17 Aug 2022 10:59:54 +0200 Subject: [PATCH 226/291] Add group call tests for muting (#2590) --- spec/test-utils/webrtc.ts | 11 +- spec/unit/webrtc/call.spec.ts | 82 +++++++++++- spec/unit/webrtc/groupCall.spec.ts | 205 ++++++++++++++++++++++++++++- src/webrtc/groupCall.ts | 5 + 4 files changed, 296 insertions(+), 7 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index d7968caa244..62bf94510bd 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -179,23 +179,26 @@ export class MockMediaStream { export class MockMediaDeviceInfo { constructor( - public kind: "audio" | "video", + public kind: "audioinput" | "videoinput" | "audiooutput", ) { } } export class MockMediaHandler { - public userMediaStreams: MediaStream[] = []; - public screensharingStreams: MediaStream[] = []; + public userMediaStreams: MockMediaStream[] = []; + public screensharingStreams: MockMediaStream[] = []; getUserMediaStream(audio: boolean, video: boolean) { const tracks = []; if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); - return new MockMediaStream("mock_stream_from_media_handler", tracks); + const stream = new MockMediaStream("mock_stream_from_media_handler", tracks); + this.userMediaStreams.push(stream); + return stream; } stopUserMediaStream() { } hasAudioDevice() { return true; } + hasVideoDevice() { return true; } stopAllStreams() {} } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 4f0e177d79a..338e968b190 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -16,7 +16,7 @@ limitations under the License. import { TestClient } from '../../TestClient'; import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call'; -import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; +import { SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; import { DUMMY_SDP, MockMediaHandler, @@ -25,6 +25,7 @@ import { installWebRTCMocks, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; +import { EventType } from "../../../src"; const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { const callPromise = call.placeVoiceCall(); @@ -34,6 +35,14 @@ const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { + const callPromise = call.placeVideoCall(); + await client.httpBackend.flush(""); + await callPromise; + + call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); +}; + describe('Call', function() { let client; let call; @@ -765,4 +774,75 @@ describe('Call', function() { expect(call.pushLocalFeed).toHaveBeenCalled(); }); }); + + describe("muting", () => { + beforeEach(async () => { + call.sendVoipEvent = jest.fn(); + await startVideoCall(client, call); + }); + + describe("sending sdp_stream_metadata_changed events", () => { + it("should send sdp_stream_metadata_changed when muting audio", async () => { + await call.setMicrophoneMuted(true); + expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: { + mock_stream_from_media_handler: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, + }, + }, + }); + }); + + it("should send sdp_stream_metadata_changed when muting video", async () => { + await call.setLocalVideoMuted(true); + expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: { + mock_stream_from_media_handler: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: false, + video_muted: true, + }, + }, + }); + }); + }); + + describe("receiving sdp_stream_metadata_changed events", () => { + const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => { + const metadata = { + stream: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }; + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track1", "video"), + ])); + call.onSDPStreamMetadataChangedReceived({ + getContent: () => ({ + [SDPStreamMetadataKey]: metadata, + }), + }); + return metadata; + }; + + it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { + const metadata = setupCall(true, false); + expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata); + expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true); + expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false); + }); + + it("should handle incoming sdp_stream_metadata_changed with video muted", async () => { + const metadata = setupCall(false, true); + expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata); + expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false); + expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true); + }); + }); + }); }); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 4bba4a91df7..378fc1de892 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -17,8 +17,16 @@ limitations under the License. import { EventType, GroupCallIntent, GroupCallType, MatrixEvent, Room, RoomMember } from '../../../src'; import { GroupCall } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; -import { installWebRTCMocks, MockMediaHandler, MockRTCPeerConnection } from '../../test-utils/webrtc'; -import { ReEmitter } from '../../../src/ReEmitter'; +import { + installWebRTCMocks, + MockMediaHandler, + MockMediaStream, + MockMediaStreamTrack, + MockRTCPeerConnection, +} from '../../test-utils/webrtc'; +import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; +import { sleep } from "../../../src/utils"; +import { ReEmitter } from "../../../src/ReEmitter"; import { TypedEventEmitter } from '../../../src/models/typed-event-emitter'; import { MediaHandler } from '../../../src/webrtc/mediaHandler'; @@ -31,6 +39,60 @@ const FAKE_SESSION_ID_1 = "alice1"; const FAKE_USER_ID_2 = "@bob:test.dummy"; const FAKE_DEVICE_ID_2 = "@BBBBBB"; const FAKE_SESSION_ID_2 = "bob1"; +const FAKE_STATE_EVENTS = [ + { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + }), + getStateKey: () => FAKE_USER_ID_1, + getRoomId: () => FAKE_ROOM_ID, + }, + { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + ["m.calls"]: [{ + ["m.call_id"]: FAKE_CONF_ID, + ["m.devices"]: [{ + device_id: FAKE_DEVICE_ID_2, + feeds: [], + }], + }], + }), + getStateKey: () => FAKE_USER_ID_2, + getRoomId: () => FAKE_ROOM_ID, + }, { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + ["m.calls"]: [{ + ["m.call_id"]: FAKE_CONF_ID, + ["m.devices"]: [{ + device_id: "user3_device", + feeds: [], + }], + }], + }), + getStateKey: () => "user3", + getRoomId: () => FAKE_ROOM_ID, + }, +]; + +const ONE_HOUR = 1000 * 60 * 60; + +const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise => { + const groupCall = new GroupCall( + cli, + room, + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + FAKE_CONF_ID, + ); + + await groupCall.create(); + await groupCall.enter(); + + return groupCall; +}; class MockCallMatrixClient { public mediaHandler: MediaHandler = new MockMediaHandler() as unknown as MediaHandler; @@ -47,6 +109,7 @@ class MockCallMatrixClient { }; sendStateEvent = jest.fn(); + sendToDevice = jest.fn(); getMediaHandler() { return this.mediaHandler; } @@ -318,4 +381,142 @@ describe('Group Call', function() { } }); }); + + describe("muting", () => { + let mockClient: MatrixClient; + let room: Room; + + beforeEach(() => { + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { + return type === EventType.GroupCallMemberPrefix + ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS + : { getContent: () => ([]) }; + }); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + }); + + describe("local muting", () => { + it("should mute local audio when calling setMicrophoneMuted()", async () => { + const groupCall = await createAndEnterGroupCall(mockClient, room); + + groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); + const setAVMutedArray = groupCall.calls.map(call => { + call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); + return call.localUsermediaFeed.setAudioVideoMuted; + }); + const tracksArray = groupCall.calls.reduce((acc, call) => { + acc.push(...call.localUsermediaStream.getAudioTracks()); + return acc; + }, []); + const sendMetadataUpdateArray = groupCall.calls.map(call => { + call.sendMetadataUpdate = jest.fn(); + return call.sendMetadataUpdate; + }); + + await groupCall.setMicrophoneMuted(true); + + groupCall.localCallFeed.stream.getAudioTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(true, null); + setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(true, null)); + tracksArray.forEach(track => expect(track.enabled).toBe(false)); + sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + + it("should mute local video when calling setLocalVideoMuted()", async () => { + const groupCall = await createAndEnterGroupCall(mockClient, room); + + groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); + const setAVMutedArray = groupCall.calls.map(call => { + call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); + return call.localUsermediaFeed.setAudioVideoMuted; + }); + const tracksArray = groupCall.calls.reduce((acc, call) => { + acc.push(...call.localUsermediaStream.getVideoTracks()); + return acc; + }, []); + const sendMetadataUpdateArray = groupCall.calls.map(call => { + call.sendMetadataUpdate = jest.fn(); + return call.sendMetadataUpdate; + }); + + await groupCall.setLocalVideoMuted(true); + + groupCall.localCallFeed.stream.getVideoTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(null, true); + setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(null, true)); + tracksArray.forEach(track => expect(track.enabled).toBe(false)); + sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + }); + + describe("remote muting", () => { + const getMetadataEvent = (audio: boolean, video: boolean): MatrixEvent => ({ + getContent: () => ({ + [SDPStreamMetadataKey]: { + stream: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }, + }), + } as MatrixEvent); + + it("should mute remote feed's audio after receiving metadata with video audio", async () => { + const metadataEvent = getMetadataEvent(true, false); + const groupCall = await createAndEnterGroupCall(mockClient, room); + + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("audio_track", "audio"), + new MockMediaStreamTrack("video_track", "video"), + ])); + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + const feed = groupCall.getUserMediaFeedByUserId(call.invitee); + expect(feed.isAudioMuted()).toBe(true); + expect(feed.isVideoMuted()).toBe(false); + + groupCall.terminate(); + }); + + it("should mute remote feed's video after receiving metadata with video muted", async () => { + const metadataEvent = getMetadataEvent(false, true); + const groupCall = await createAndEnterGroupCall(mockClient, room); + + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("audio_track", "audio"), + new MockMediaStreamTrack("video_track", "video"), + ])); + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + const feed = groupCall.getUserMediaFeedByUserId(call.invitee); + expect(feed.isAudioMuted()).toBe(false); + expect(feed.isVideoMuted()).toBe(true); + + groupCall.terminate(); + }); + }); + }); }); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 83239fadeae..879bc49bee9 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -373,6 +373,11 @@ export class GroupCall extends TypedEventEmitter< this.retryCallCounts.clear(); clearTimeout(this.retryCallLoopTimeout); + for (const [userId] of this.memberStateExpirationTimers) { + clearTimeout(this.memberStateExpirationTimers.get(userId)); + this.memberStateExpirationTimers.delete(userId); + } + if (this.transmitTimer !== null) { clearTimeout(this.transmitTimer); this.transmitTimer = null; From e4cf5b26eeeb55ee9dd12c5ca382d15a8474b004 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 17 Aug 2022 12:33:11 +0100 Subject: [PATCH 227/291] Add test for updateLocalUsermediaStream (#2596) --- spec/test-utils/webrtc.ts | 5 ++++- spec/unit/webrtc/groupCall.spec.ts | 27 +++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 62bf94510bd..28ee7e8a812 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -148,6 +148,7 @@ export class MockMediaStream { ) {} listeners: [string, (...args: any[]) => any][] = []; + public isStopped = false; dispatchEvent(eventType: string) { this.listeners.forEach(([t, c]) => { @@ -196,7 +197,9 @@ export class MockMediaHandler { this.userMediaStreams.push(stream); return stream; } - stopUserMediaStream() { } + stopUserMediaStream(stream: MockMediaStream) { + stream.isStopped = true; + } hasAudioDevice() { return true; } hasVideoDevice() { return true; } stopAllStreams() {} diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 378fc1de892..6fc7ddf009c 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -263,6 +263,33 @@ describe('Group Call', function() { groupCall.leave(); } }); + + it("retains state of local user media stream when updated", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + const oldStream = groupCall.localCallFeed.stream as unknown as MockMediaStream; + + // arbitrary values, important part is that they're the same afterwards + await groupCall.setLocalVideoMuted(true); + await groupCall.setMicrophoneMuted(false); + + const newStream = await mockClient.getMediaHandler().getUserMediaStream(true, true); + + groupCall.updateLocalUsermediaStream(newStream); + + expect(groupCall.localCallFeed.stream).toBe(newStream); + + expect(groupCall.isLocalVideoMuted()).toEqual(true); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + + expect(oldStream.isStopped).toEqual(true); + } finally { + groupCall.leave(); + } + }); }); describe('Placing calls', function() { From 2566c40e969431fd7c19bd8ab19be337c7505cec Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 17 Aug 2022 15:10:03 +0100 Subject: [PATCH 228/291] Add tests for incoming calls in group calls (#2597) * Add tests for incoming calls in group calls Inspiration wwlecome for the renamed describe group which we're really abusing for a bunch of things that happen to have the same dependencies. Fixes https://github.com/vector-im/element-call/issues/532 * Extract incoming call tests out into their own describe and get the lexicographical ordering to match who should be calling who * Trailing space --- spec/unit/webrtc/groupCall.spec.ts | 107 +++++++++++++++++++++++++++-- 1 file changed, 102 insertions(+), 5 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 6fc7ddf009c..5299948329d 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { EventType, GroupCallIntent, GroupCallType, MatrixEvent, Room, RoomMember } from '../../../src'; +import { EventType, GroupCallIntent, GroupCallType, MatrixCall, MatrixEvent, Room, RoomMember } from '../../../src'; import { GroupCall } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; import { @@ -29,6 +29,9 @@ import { sleep } from "../../../src/utils"; import { ReEmitter } from "../../../src/ReEmitter"; import { TypedEventEmitter } from '../../../src/models/typed-event-emitter'; import { MediaHandler } from '../../../src/webrtc/mediaHandler'; +import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from '../../../src/webrtc/callEventHandler'; +import { CallFeed } from '../../../src/webrtc/callFeed'; +import { CallState } from '../../../src/webrtc/call'; const FAKE_ROOM_ID = "!fake:test.dummy"; const FAKE_CONF_ID = "fakegroupcallid"; @@ -94,10 +97,11 @@ const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise { public mediaHandler: MediaHandler = new MockMediaHandler() as unknown as MediaHandler; constructor(public userId: string, public deviceId: string, public sessionId: string) { + super(); } groupCallEventHandler = { @@ -118,9 +122,6 @@ class MockCallMatrixClient { getDeviceId() { return this.deviceId; } getSessionId() { return this.sessionId; } - emit = jest.fn(); - on = jest.fn(); - removeListener = jest.fn(); getTurnServers = () => []; isFallbackICEServerAllowed = () => false; reEmitter = new ReEmitter(new TypedEventEmitter()); @@ -128,6 +129,28 @@ class MockCallMatrixClient { checkTurnServers = () => null; } +class MockCall { + constructor(public roomId: string, public groupCallId: string) { + } + + public state = CallState.Ringing; + public opponentUserId = FAKE_USER_ID_1; + public callId = "1"; + + public reject = jest.fn(); + public answerWithCallFeeds = jest.fn(); + public hangup = jest.fn(); + + on = jest.fn(); + removeListener = jest.fn(); + + getOpponentMember() { + return { + userId: this.opponentUserId, + }; + } +} + describe('Group Call', function() { beforeEach(function() { installWebRTCMocks(); @@ -546,4 +569,78 @@ describe('Group Call', function() { }); }); }); + + describe("incoming calls", () => { + let mockClient: MatrixClient; + let room: Room; + let groupCall: GroupCall; + + beforeEach(async () => { + // we are bob here because we're testing incoming calls, and since alice's user id + // is lexicographically before Bob's, the spec requires that she calls Bob. + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_2, FAKE_DEVICE_ID_2, FAKE_SESSION_ID_2, + ); + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_2); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + + groupCall = await createAndEnterGroupCall(mockClient, room); + }); + + afterEach(() => { + groupCall.leave(); + }); + + it("ignores incoming calls for other rooms", async () => { + const mockCall = new MockCall("!someotherroom.fake.dummy", groupCall.groupCallId); + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).not.toHaveBeenCalled(); + expect(mockCall.answerWithCallFeeds).not.toHaveBeenCalled(); + }); + + it("rejects incoming calls for the wrong group call", async () => { + const mockCall = new MockCall(room.roomId, "not " + groupCall.groupCallId); + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).toHaveBeenCalled(); + }); + + it("ignores incoming calls not in the ringing state", async () => { + const mockCall = new MockCall(room.roomId, groupCall.groupCallId); + mockCall.state = CallState.Connected; + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).not.toHaveBeenCalled(); + expect(mockCall.answerWithCallFeeds).not.toHaveBeenCalled(); + }); + + it("answers calls for the right room & group call ID", async () => { + const mockCall = new MockCall(room.roomId, groupCall.groupCallId); + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).not.toHaveBeenCalled(); + expect(mockCall.answerWithCallFeeds).toHaveBeenCalled(); + expect(groupCall.calls).toEqual([mockCall]); + }); + + it("replaces calls if it already has one with the same user", async () => { + const oldMockCall = new MockCall(room.roomId, groupCall.groupCallId); + const newMockCall = new MockCall(room.roomId, groupCall.groupCallId); + newMockCall.callId = "not " + oldMockCall.callId; + + mockClient.emit(CallEventHandlerEvent.Incoming, oldMockCall as unknown as MatrixCall); + mockClient.emit(CallEventHandlerEvent.Incoming, newMockCall as unknown as MatrixCall); + + expect(oldMockCall.hangup).toHaveBeenCalled(); + expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled(); + expect(groupCall.calls).toEqual([newMockCall]); + }); + }); }); From 9589a97952178c519ef5c43ff9b256cb10ccd0ca Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 17 Aug 2022 18:09:46 +0100 Subject: [PATCH 229/291] Test muting in PTT mode (#2599) Fixes https://github.com/vector-im/element-call/issues/523 --- spec/unit/webrtc/groupCall.spec.ts | 121 +++++++++++++++++++++++++---- 1 file changed, 104 insertions(+), 17 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 5299948329d..161062720f3 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -136,11 +136,17 @@ class MockCall { public state = CallState.Ringing; public opponentUserId = FAKE_USER_ID_1; public callId = "1"; + public localUsermediaFeed = { + setAudioVideoMuted: jest.fn(), + stream: new MockMediaStream("stream"), + }; public reject = jest.fn(); public answerWithCallFeeds = jest.fn(); public hangup = jest.fn(); + public sendMetadataUpdate = jest.fn(); + on = jest.fn(); removeListener = jest.fn(); @@ -230,23 +236,6 @@ describe('Group Call', function() { } }); - it("starts with mic muted in PTT calls", async () => { - try { - // replace groupcall with a PTT one for this test - // we will probably want a dedicated test suite for PTT calls, so when we do, - // this can go in there instead. - groupCall = new GroupCall(mockClient, room, GroupCallType.Video, true, GroupCallIntent.Prompt); - - await groupCall.create(); - - await groupCall.initLocalCallFeed(); - - expect(groupCall.isMicrophoneMuted()).toEqual(true); - } finally { - groupCall.leave(); - } - }); - it("disables audio stream when audio is set to muted", async () => { try { await groupCall.create(); @@ -313,6 +302,104 @@ describe('Group Call', function() { groupCall.leave(); } }); + + describe("PTT calls", () => { + beforeEach(async () => { + // replace groupcall with a PTT one + groupCall = new GroupCall(mockClient, room, GroupCallType.Video, true, GroupCallIntent.Prompt); + + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + }); + + afterEach(() => { + jest.useRealTimers(); + + groupCall.leave(); + }); + + it("starts with mic muted in PTT calls", async () => { + expect(groupCall.isMicrophoneMuted()).toEqual(true); + }); + + it("re-mutes microphone after transmit timeout in PTT mode", async () => { + jest.useFakeTimers(); + + await groupCall.setMicrophoneMuted(false); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + + jest.advanceTimersByTime(groupCall.pttMaxTransmitTime + 100); + + expect(groupCall.isMicrophoneMuted()).toEqual(true); + }); + + it("timer is cleared when mic muted again in PTT mode", async () => { + jest.useFakeTimers(); + + await groupCall.setMicrophoneMuted(false); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + + // 'talk' for half the allowed time + jest.advanceTimersByTime(groupCall.pttMaxTransmitTime / 2); + + await groupCall.setMicrophoneMuted(true); + await groupCall.setMicrophoneMuted(false); + + // we should still be unmuted after almost the full timeout duration + // if not, the timer for the original talking session must have fired + jest.advanceTimersByTime(groupCall.pttMaxTransmitTime - 100); + + expect(groupCall.isMicrophoneMuted()).toEqual(false); + }); + + it("sends metadata updates before unmuting in PTT mode", async () => { + const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); + groupCall.calls.push(mockCall as unknown as MatrixCall); + + let metadataUpdateResolve: () => void; + const metadataUpdatePromise = new Promise(resolve => { + metadataUpdateResolve = resolve; + }); + mockCall.sendMetadataUpdate = jest.fn().mockReturnValue(metadataUpdatePromise); + + const mutePromise = groupCall.setMicrophoneMuted(false); + // we should still be muted at this point because the metadata update hasn't sent + expect(groupCall.isMicrophoneMuted()).toEqual(true); + expect(mockCall.localUsermediaFeed.setAudioVideoMuted).not.toHaveBeenCalled(); + metadataUpdateResolve(); + + await mutePromise; + + expect(mockCall.localUsermediaFeed.setAudioVideoMuted).toHaveBeenCalled(); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + }); + + it("sends metadata updates after muting in PTT mode", async () => { + const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); + groupCall.calls.push(mockCall as unknown as MatrixCall); + + // the call starts muted, so unmute to get in the right state to test + await groupCall.setMicrophoneMuted(false); + mockCall.localUsermediaFeed.setAudioVideoMuted.mockReset(); + + let metadataUpdateResolve: () => void; + const metadataUpdatePromise = new Promise(resolve => { + metadataUpdateResolve = resolve; + }); + mockCall.sendMetadataUpdate = jest.fn().mockReturnValue(metadataUpdatePromise); + + const mutePromise = groupCall.setMicrophoneMuted(true); + // we should be muted at this point, before the metadata update has been sent + expect(groupCall.isMicrophoneMuted()).toEqual(true); + expect(mockCall.localUsermediaFeed.setAudioVideoMuted).toHaveBeenCalled(); + metadataUpdateResolve(); + + await mutePromise; + + expect(groupCall.isMicrophoneMuted()).toEqual(true); + }); + }); }); describe('Placing calls', function() { From 448a5c9a778d239a0f6c8b42c6865333e98e75bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 18 Aug 2022 10:42:03 +0200 Subject: [PATCH 230/291] Add screensharing tests (#2598) --- spec/test-utils/webrtc.ts | 34 ++++++++++++++ spec/unit/webrtc/groupCall.spec.ts | 73 ++++++++++++++++++++++++++++++ src/webrtc/groupCall.ts | 4 +- 3 files changed, 110 insertions(+), 1 deletion(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 28ee7e8a812..9ed142fcc00 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { IScreensharingOpts } from "../../src/webrtc/mediaHandler"; + export const DUMMY_SDP = ( "v=0\r\n" + "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" + @@ -75,6 +77,7 @@ export class MockRTCPeerConnection { private negotiationNeededListener: () => void; private needsNegotiation = false; localDescription: RTCSessionDescription; + signalingState: RTCSignalingState = "stable"; public static triggerAllNegotiations() { for (const inst of this.instances) { @@ -137,6 +140,26 @@ export class MockMediaStreamTrack { constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { } stop() { } + + listeners: [string, (...args: any[]) => any][] = []; + public isStopped = false; + + // XXX: Using EventTarget in jest doesn't seem to work, so we write our own + // implementation + dispatchEvent(eventType: string) { + this.listeners.forEach(([t, c]) => { + if (t !== eventType) return; + c(); + }); + } + addEventListener(eventType: string, callback: (...args: any[]) => any) { + this.listeners.push([eventType, callback]); + } + removeEventListener(eventType: string, callback: (...args: any[]) => any) { + this.listeners.filter(([t, c]) => { + return t !== eventType || c !== callback; + }); + } } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own @@ -200,6 +223,17 @@ export class MockMediaHandler { stopUserMediaStream(stream: MockMediaStream) { stream.isStopped = true; } + getScreensharingStream(opts?: IScreensharingOpts) { + const tracks = [new MockMediaStreamTrack("video_track", "video")]; + if (opts?.audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); + + const stream = new MockMediaStream("mock_screen_stream_from_media_handler", tracks); + this.screensharingStreams.push(stream); + return stream; + } + stopScreensharingStream(stream: MockMediaStream) { + stream.isStopped = true; + } hasAudioDevice() { return true; } hasVideoDevice() { return true; } stopAllStreams() {} diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 161062720f3..0423930ae2f 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -730,4 +730,77 @@ describe('Group Call', function() { expect(groupCall.calls).toEqual([newMockCall]); }); }); + + describe("screensharing", () => { + let mockClient: MatrixClient; + let room: Room; + let groupCall: GroupCall; + + beforeEach(async () => { + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { + return type === EventType.GroupCallMemberPrefix + ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS + : { getContent: () => ([]) }; + }); + + groupCall = await createAndEnterGroupCall(mockClient, room); + }); + + it("sending screensharing stream", async () => { + const onNegotiationNeededArray = groupCall.calls.map(call => { + // @ts-ignore Mock + call.gotLocalOffer = jest.fn(); + // @ts-ignore Mock + return call.gotLocalOffer; + }); + + await groupCall.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + + expect(groupCall.screenshareFeeds).toHaveLength(1); + groupCall.calls.forEach(c => { + expect(c.getLocalFeeds().find(f => f.purpose === SDPStreamMetadataPurpose.Screenshare)).toBeDefined(); + }); + onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + + it("receiving screensharing stream", async () => { + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + call.onNegotiateReceived({ + getContent: () => ({ + [SDPStreamMetadataKey]: { + "screensharing_stream": { + purpose: SDPStreamMetadataPurpose.Screenshare, + }, + }, + description: { + type: "offer", + sdp: "...", + }, + }), + } as MatrixEvent); + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("screensharing_stream", [ + new MockMediaStreamTrack("video_track", "video"), + ])); + + expect(groupCall.screenshareFeeds).toHaveLength(1); + expect(groupCall.getScreenshareFeedByUserId(call.invitee)).toBeDefined(); + + groupCall.terminate(); + }); + }); }); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 879bc49bee9..6672dbd539b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -182,7 +182,7 @@ export class GroupCall extends TypedEventEmitter< private reEmitter: ReEmitter; private transmitTimer: ReturnType | null = null; private memberStateExpirationTimers: Map> = new Map(); - private resendMemberStateTimer: ReturnType | null = null; + private resendMemberStateTimer: ReturnType | null = null; constructor( private client: MatrixClient, @@ -698,6 +698,8 @@ export class GroupCall extends TypedEventEmitter< const res = await send(); + // Clear the old interval first, so that it isn't forgot + clearInterval(this.resendMemberStateTimer); // Resend the state event every so often so it doesn't become stale this.resendMemberStateTimer = setInterval(async () => { logger.log("Resending call member state"); From e95947dc7334e9248fadd0468a92b963b0b408a5 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 19 Aug 2022 08:22:37 -0400 Subject: [PATCH 231/291] Update lockfile (#2603) --- yarn.lock | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/yarn.lock b/yarn.lock index d20923e47b9..856ba07e5cf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6827,6 +6827,11 @@ universal-user-agent@^6.0.0: resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== +universalify@^0.1.2: + version "0.1.2" + resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" + integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== + update-browserslist-db@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/update-browserslist-db/-/update-browserslist-db-1.0.5.tgz#be06a5eedd62f107b7c19eb5bcefb194411abf38" From 45e56f8cc36c459ed43e405be4206e5e66b3ad98 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 19 Aug 2022 17:33:55 +0200 Subject: [PATCH 232/291] Add `disposed` to `CallFeed` (#2604) --- src/webrtc/callFeed.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 44a568cea58..d10c1c2cadb 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -47,6 +47,7 @@ export enum CallFeedEvent { LocalVolumeChanged = "local_volume_changed", VolumeChanged = "volume_changed", Speaking = "speaking", + Disposed = "disposed", } type EventHandlerMap = { @@ -55,6 +56,7 @@ type EventHandlerMap = { [CallFeedEvent.LocalVolumeChanged]: (localVolume: number) => void; [CallFeedEvent.VolumeChanged]: (volume: number) => void; [CallFeedEvent.Speaking]: (speaking: boolean) => void; + [CallFeedEvent.Disposed]: () => void; }; export class CallFeed extends TypedEventEmitter { @@ -76,6 +78,7 @@ export class CallFeed extends TypedEventEmitter private speakingThreshold = SPEAKING_THRESHOLD; private speaking = false; private volumeLooperTimeout: ReturnType; + private _disposed = false; constructor(opts: ICallFeedOpts) { super(); @@ -295,6 +298,16 @@ export class CallFeed extends TypedEventEmitter this.analyser = null; releaseContext(); } + this._disposed = true; + this.emit(CallFeedEvent.Disposed); + } + + public get disposed(): boolean { + return this._disposed; + } + + private set disposed(value: boolean) { + this._disposed = value; } public getLocalVolume(): number { From 92cd84fc0c7a7b445af98bb5bc5bcf8ca734cc0b Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 22 Aug 2022 13:29:54 +0100 Subject: [PATCH 233/291] Add unit tests for hangup / reject (#2606) * Add unit tests for hangup / reject Fixes https://github.com/vector-im/element-call/issues/537 * Fix some bugs where we carried on with the call after it had been ended --- spec/test-utils/webrtc.ts | 6 ++ spec/unit/webrtc/call.spec.ts | 124 +++++++++++++++++++++++++++------- src/webrtc/call.ts | 7 ++ 3 files changed, 112 insertions(+), 25 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 9ed142fcc00..80ccbef2668 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -109,6 +109,12 @@ export class MockRTCPeerConnection { sdp: DUMMY_SDP, }); } + createAnswer() { + return Promise.resolve({ + type: 'answer', + sdp: DUMMY_SDP, + }); + } setRemoteDescription() { return Promise.resolve(); } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 338e968b190..1cedd5c7cc4 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -25,7 +25,9 @@ import { installWebRTCMocks, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; -import { EventType } from "../../../src"; +import { EventType, MatrixEvent } from "../../../src"; + +const FAKE_ROOM_ID = "!foo:bar"; const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { const callPromise = call.placeVoiceCall(); @@ -43,6 +45,31 @@ const startVideoCall = async (client: TestClient, call: MatrixCall): Promise { + const callPromise = call.initWithInvite({ + getContent: jest.fn().mockReturnValue({ + version, + call_id: "call_id", + party_id: "remote_party_id", + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + getLocalAge: () => null, + } as unknown as MatrixEvent); + call.getFeeds().push(new CallFeed({ + client: client.client, + userId: "remote_user_id", + // @ts-ignore Mock + stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), + id: "remote_feed_id", + purpose: SDPStreamMetadataPurpose.Usermedia, + })); + await client.httpBackend.flush(""); + await callPromise; +}; + describe('Call', function() { let client; let call; @@ -60,7 +87,7 @@ describe('Call', function() { client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); // We just stub out sendEvent: we're not interested in testing the client's // event sending code here - client.client.sendEvent = () => {}; + client.client.sendEvent = jest.fn(); client.client.mediaHandler = new MockMediaHandler; client.client.getMediaHandler = () => client.client.mediaHandler; client.httpBackend.when("GET", "/voip/turnServer").respond(200, {}); @@ -74,7 +101,7 @@ describe('Call', function() { call = new MatrixCall({ client: client.client, - roomId: '!foo:bar', + roomId: FAKE_ROOM_ID, }); // call checks one of these is wired up call.on('error', () => {}); @@ -594,28 +621,7 @@ describe('Call', function() { }); it("should end call after receiving a select event with a different party id", async () => { - const callPromise = call.initWithInvite({ - getContent: () => ({ - version: 1, - call_id: "call_id", - party_id: "remote_party_id", - offer: { - sdp: DUMMY_SDP, - }, - }), - getSender: () => "@test:foo", - getLocalAge: () => null, - }); - call.feeds.push(new CallFeed({ - client, - userId: "remote_user_id", - // @ts-ignore Mock - stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), - id: "remote_feed_id", - purpose: SDPStreamMetadataPurpose.Usermedia, - })); - await client.httpBackend.flush(); - await callPromise; + await fakeIncomingCall(client, call); const callHangupCallback = jest.fn(); call.on(CallEvent.Hangup, callHangupCallback); @@ -845,4 +851,72 @@ describe('Call', function() { }); }); }); + + describe("rejecting calls", () => { + it("sends hangup event when rejecting v0 calls", async () => { + await fakeIncomingCall(client, call, 0); + + call.reject(); + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallHangup, + expect.objectContaining({ + call_id: call.callId, + }), + ); + }); + + it("sends reject event when rejecting v1 calls", async () => { + await fakeIncomingCall(client, call, "1"); + + call.reject(); + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallReject, + expect.objectContaining({ + call_id: call.callId, + }), + ); + }); + + it("does not reject a call that has already been answered", async () => { + await fakeIncomingCall(client, call, "1"); + + await call.answer(); + + client.client.sendEvent.mockReset(); + + let caught = false; + try { + call.reject(); + } catch (e) { + caught = true; + } + + expect(caught).toEqual(true); + expect(client.client.sendEvent).not.toHaveBeenCalled(); + + call.hangup(); + }); + + it("hangs up a call", async () => { + await fakeIncomingCall(client, call, "1"); + + await call.answer(); + + client.client.sendEvent.mockReset(); + + call.hangup(); + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallHangup, + expect.objectContaining({ + call_id: call.callId, + }), + ); + }); + }); }); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 43ef311882f..8e14e06a880 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -1569,6 +1569,10 @@ export class MatrixCall extends TypedEventEmitter Date: Mon, 22 Aug 2022 20:17:19 +0100 Subject: [PATCH 234/291] Test call timeouts (#2611) Fixes https://github.com/vector-im/element-call/issues/547 --- spec/unit/webrtc/call.spec.ts | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 1cedd5c7cc4..f7c51d10c53 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -15,7 +15,14 @@ limitations under the License. */ import { TestClient } from '../../TestClient'; -import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call'; +import { + MatrixCall, + CallErrorCode, + CallEvent, + supportsMatrixCall, + CallType, + CallState, +} from '../../../src/webrtc/call'; import { SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; import { DUMMY_SDP, @@ -28,6 +35,7 @@ import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, MatrixEvent } from "../../../src"; const FAKE_ROOM_ID = "!foo:bar"; +const CALL_LIFETIME = 60000; const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { const callPromise = call.placeVoiceCall(); @@ -51,12 +59,13 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s version, call_id: "call_id", party_id: "remote_party_id", + lifetime: CALL_LIFETIME, offer: { sdp: DUMMY_SDP, }, }), getSender: () => "@test:foo", - getLocalAge: () => null, + getLocalAge: () => 1, } as unknown as MatrixEvent); call.getFeeds().push(new CallFeed({ client: client.client, @@ -66,7 +75,6 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s id: "remote_feed_id", purpose: SDPStreamMetadataPurpose.Usermedia, })); - await client.httpBackend.flush(""); await callPromise; }; @@ -90,6 +98,7 @@ describe('Call', function() { client.client.sendEvent = jest.fn(); client.client.mediaHandler = new MockMediaHandler; client.client.getMediaHandler = () => client.client.mediaHandler; + client.client.turnServersExpiry = Date.now() + 60 * 60 * 1000; client.httpBackend.when("GET", "/voip/turnServer").respond(200, {}); client.client.getRoom = () => { return { @@ -112,6 +121,8 @@ describe('Call', function() { global.navigator = prevNavigator; global.window = prevWindow; global.document = prevDocument; + + jest.useRealTimers(); }); it('should ignore candidate events from non-matching party ID', async function() { @@ -919,4 +930,15 @@ describe('Call', function() { ); }); }); + + it("times out an incoming call", async () => { + jest.useFakeTimers(); + await fakeIncomingCall(client, call, "1"); + + expect(call.state).toEqual(CallState.Ringing); + + jest.advanceTimersByTime(CALL_LIFETIME + 1000); + + expect(call.state).toEqual(CallState.Ended); + }); }); From be94f5ea9332ded2366a2fd47bbded242a66efaa Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 22 Aug 2022 20:30:27 +0100 Subject: [PATCH 235/291] Fix imports --- src/client.ts | 1 - src/sliding-sync-sdk.ts | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/client.ts b/src/client.ts index 51966f6eff4..3ec0dcb8590 100644 --- a/src/client.ts +++ b/src/client.ts @@ -122,7 +122,6 @@ import { CrossSigningInfo, DeviceTrustLevel, ICacheCallbacks, UserTrustLevel } f import { Room, NotificationCountType, RoomEvent, RoomEventHandlerMap, RoomNameState } from "./models/room"; import { RoomMemberEvent, RoomMemberEventHandlerMap } from "./models/room-member"; import { RoomStateEvent, RoomStateEventHandlerMap } from "./models/room-state"; - import { IAddThreePidOnlyBody, IBindThreePidBody, diff --git a/src/sliding-sync-sdk.ts b/src/sliding-sync-sdk.ts index b9cd4d40b91..46f6c5f6c4d 100644 --- a/src/sliding-sync-sdk.ts +++ b/src/sliding-sync-sdk.ts @@ -36,6 +36,8 @@ import { import { EventType } from "./@types/event"; import { IPushRules } from "./@types/PushRules"; import { PushProcessor } from "./pushprocessor"; +import { RoomStateEvent } from "./models/room-state"; +import { RoomMemberEvent } from "./models/room-member"; // Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed // to RECONNECTING. This is needed to inform the client of server issues when the From 4a294c9dd36109233041ede34e780685c5d69801 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 23 Aug 2022 22:02:32 +0100 Subject: [PATCH 236/291] 1:1 screenshare tests Fixes https://github.com/vector-im/element-call/issues/548 --- spec/test-utils/webrtc.ts | 11 ++++- spec/unit/webrtc/call.spec.ts | 82 +++++++++++++++++++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 80ccbef2668..e2118efa466 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -56,6 +56,9 @@ export const DUMMY_SDP = ( "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" ); +export const USERMEDIA_STREAM_ID = "mock_stream_from_media_handler"; +export const SCREENSHARE_STREAM_ID = "mock_screen_stream_from_media_handler"; + class MockMediaStreamAudioSourceNode { connect() {} } @@ -128,6 +131,10 @@ export class MockRTCPeerConnection { return new MockRTCRtpSender(track); } + removeTrack() { + this.needsNegotiation = true; + } + doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { this.needsNegotiation = false; @@ -222,7 +229,7 @@ export class MockMediaHandler { if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); - const stream = new MockMediaStream("mock_stream_from_media_handler", tracks); + const stream = new MockMediaStream(USERMEDIA_STREAM_ID, tracks); this.userMediaStreams.push(stream); return stream; } @@ -233,7 +240,7 @@ export class MockMediaHandler { const tracks = [new MockMediaStreamTrack("video_track", "video")]; if (opts?.audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); - const stream = new MockMediaStream("mock_screen_stream_from_media_handler", tracks); + const stream = new MockMediaStream(SCREENSHARE_STREAM_ID, tracks); this.screensharingStreams.push(stream); return stream; } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index f7c51d10c53..88213821dee 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -30,6 +30,8 @@ import { MockMediaStream, MockMediaStreamTrack, installWebRTCMocks, + MockRTCPeerConnection, + SCREENSHARE_STREAM_ID, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, MatrixEvent } from "../../../src"; @@ -941,4 +943,84 @@ describe('Call', function() { expect(call.state).toEqual(CallState.Ended); }); + + describe("Screen sharing", () => { + beforeEach(async () => { + await startVoiceCall(client, call); + + await call.onAnswerReceived({ + getContent: () => { + return { + "version": 1, + "call_id": call.callId, + "party_id": 'party_id', + "answer": { + sdp: DUMMY_SDP, + }, + "org.matrix.msc3077.sdp_stream_metadata": { + "foo": { + "purpose": "m.usermedia", + "audio_muted": false, + "video_muted": false, + }, + }, + }; + }, + getSender: () => "@test:foo", + }); + }); + + afterEach(() => { + // Hangup to stop timers + call.hangup(CallErrorCode.UserHangup, true); + }); + + it("enables screensharing", async () => { + await call.setScreensharingEnabled(true); + + expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(1); + + client.client.sendEvent.mockReset(); + const sendNegotiatePromise = new Promise(resolve => { + client.client.sendEvent.mockImplementation(() => { + resolve(); + }); + }); + + MockRTCPeerConnection.triggerAllNegotiations(); + await sendNegotiatePromise; + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallNegotiate, + expect.objectContaining({ + "version": "1", + "call_id": call.callId, + "org.matrix.msc3077.sdp_stream_metadata": expect.objectContaining({ + [SCREENSHARE_STREAM_ID]: expect.objectContaining({ + purpose: SDPStreamMetadataPurpose.Screenshare, + }), + }), + }), + ); + }); + + it("disables screensharing", async () => { + await call.setScreensharingEnabled(true); + + client.client.sendEvent.mockReset(); + const sendNegotiatePromise = new Promise(resolve => { + client.client.sendEvent.mockImplementation(() => { + resolve(); + }); + }); + + MockRTCPeerConnection.triggerAllNegotiations(); + await sendNegotiatePromise; + + await call.setScreensharingEnabled(false); + + expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(0); + }); + }); }); From c527f85fb1001305a248c4ba44e6a6b0d72f689a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 23 Aug 2022 22:03:25 +0100 Subject: [PATCH 237/291] Revert 4a294c9dd36109233041ede34e780685c5d69801 Pushed to wrong branch --- spec/test-utils/webrtc.ts | 11 +---- spec/unit/webrtc/call.spec.ts | 82 ----------------------------------- 2 files changed, 2 insertions(+), 91 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index e2118efa466..80ccbef2668 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -56,9 +56,6 @@ export const DUMMY_SDP = ( "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" ); -export const USERMEDIA_STREAM_ID = "mock_stream_from_media_handler"; -export const SCREENSHARE_STREAM_ID = "mock_screen_stream_from_media_handler"; - class MockMediaStreamAudioSourceNode { connect() {} } @@ -131,10 +128,6 @@ export class MockRTCPeerConnection { return new MockRTCRtpSender(track); } - removeTrack() { - this.needsNegotiation = true; - } - doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { this.needsNegotiation = false; @@ -229,7 +222,7 @@ export class MockMediaHandler { if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); - const stream = new MockMediaStream(USERMEDIA_STREAM_ID, tracks); + const stream = new MockMediaStream("mock_stream_from_media_handler", tracks); this.userMediaStreams.push(stream); return stream; } @@ -240,7 +233,7 @@ export class MockMediaHandler { const tracks = [new MockMediaStreamTrack("video_track", "video")]; if (opts?.audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); - const stream = new MockMediaStream(SCREENSHARE_STREAM_ID, tracks); + const stream = new MockMediaStream("mock_screen_stream_from_media_handler", tracks); this.screensharingStreams.push(stream); return stream; } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 88213821dee..f7c51d10c53 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -30,8 +30,6 @@ import { MockMediaStream, MockMediaStreamTrack, installWebRTCMocks, - MockRTCPeerConnection, - SCREENSHARE_STREAM_ID, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, MatrixEvent } from "../../../src"; @@ -943,84 +941,4 @@ describe('Call', function() { expect(call.state).toEqual(CallState.Ended); }); - - describe("Screen sharing", () => { - beforeEach(async () => { - await startVoiceCall(client, call); - - await call.onAnswerReceived({ - getContent: () => { - return { - "version": 1, - "call_id": call.callId, - "party_id": 'party_id', - "answer": { - sdp: DUMMY_SDP, - }, - "org.matrix.msc3077.sdp_stream_metadata": { - "foo": { - "purpose": "m.usermedia", - "audio_muted": false, - "video_muted": false, - }, - }, - }; - }, - getSender: () => "@test:foo", - }); - }); - - afterEach(() => { - // Hangup to stop timers - call.hangup(CallErrorCode.UserHangup, true); - }); - - it("enables screensharing", async () => { - await call.setScreensharingEnabled(true); - - expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(1); - - client.client.sendEvent.mockReset(); - const sendNegotiatePromise = new Promise(resolve => { - client.client.sendEvent.mockImplementation(() => { - resolve(); - }); - }); - - MockRTCPeerConnection.triggerAllNegotiations(); - await sendNegotiatePromise; - - expect(client.client.sendEvent).toHaveBeenCalledWith( - FAKE_ROOM_ID, - EventType.CallNegotiate, - expect.objectContaining({ - "version": "1", - "call_id": call.callId, - "org.matrix.msc3077.sdp_stream_metadata": expect.objectContaining({ - [SCREENSHARE_STREAM_ID]: expect.objectContaining({ - purpose: SDPStreamMetadataPurpose.Screenshare, - }), - }), - }), - ); - }); - - it("disables screensharing", async () => { - await call.setScreensharingEnabled(true); - - client.client.sendEvent.mockReset(); - const sendNegotiatePromise = new Promise(resolve => { - client.client.sendEvent.mockImplementation(() => { - resolve(); - }); - }); - - MockRTCPeerConnection.triggerAllNegotiations(); - await sendNegotiatePromise; - - await call.setScreensharingEnabled(false); - - expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(0); - }); - }); }); From 9e1b126854d92a0aa1d444c24f95a85f052c6904 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 24 Aug 2022 15:45:53 +0100 Subject: [PATCH 238/291] 1:1 screenshare tests (#2617) * 1:1 screenshare tests Fixes https://github.com/vector-im/element-call/issues/548 * Always hang up calls after tests to prevent hanging tests Also fix a null dereference as we may not have an invitee or opponent member when sending voip events if not using to-device messages. * use mockImplementationOnce Co-authored-by: Robin * use mockImplementationOnce Co-authored-by: Robin * Add type on mock * Add corresponding call.off * Merge enable & disable screenshare tests Co-authored-by: Robin --- spec/test-utils/webrtc.ts | 11 ++++- spec/unit/webrtc/call.spec.ts | 83 +++++++++++++++++++++++++++++++---- src/webrtc/call.ts | 2 +- 3 files changed, 85 insertions(+), 11 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 80ccbef2668..e2118efa466 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -56,6 +56,9 @@ export const DUMMY_SDP = ( "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" ); +export const USERMEDIA_STREAM_ID = "mock_stream_from_media_handler"; +export const SCREENSHARE_STREAM_ID = "mock_screen_stream_from_media_handler"; + class MockMediaStreamAudioSourceNode { connect() {} } @@ -128,6 +131,10 @@ export class MockRTCPeerConnection { return new MockRTCRtpSender(track); } + removeTrack() { + this.needsNegotiation = true; + } + doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { this.needsNegotiation = false; @@ -222,7 +229,7 @@ export class MockMediaHandler { if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); - const stream = new MockMediaStream("mock_stream_from_media_handler", tracks); + const stream = new MockMediaStream(USERMEDIA_STREAM_ID, tracks); this.userMediaStreams.push(stream); return stream; } @@ -233,7 +240,7 @@ export class MockMediaHandler { const tracks = [new MockMediaStreamTrack("video_track", "video")]; if (opts?.audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); - const stream = new MockMediaStream("mock_screen_stream_from_media_handler", tracks); + const stream = new MockMediaStream(SCREENSHARE_STREAM_ID, tracks); this.screensharingStreams.push(stream); return stream; } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index f7c51d10c53..cac83f4a832 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -30,6 +30,8 @@ import { MockMediaStream, MockMediaStreamTrack, installWebRTCMocks, + MockRTCPeerConnection, + SCREENSHARE_STREAM_ID, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, MatrixEvent } from "../../../src"; @@ -117,6 +119,9 @@ describe('Call', function() { }); afterEach(function() { + // Hangup to stop timers + call.hangup(CallErrorCode.UserHangup, true); + client.stop(); global.navigator = prevNavigator; global.window = prevWindow; @@ -178,9 +183,6 @@ describe('Call', function() { getSender: () => "@test:foo", }); expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); - - // Hangup to stop timers - call.hangup(CallErrorCode.UserHangup, true); }); it('should add candidates received before answer if party ID is correct', async function() { @@ -283,9 +285,6 @@ describe('Call', function() { const ident = call.getRemoteAssertedIdentity(); expect(ident.id).toEqual("@steve:example.com"); expect(ident.displayName).toEqual("Steve Gibbons"); - - // Hangup to stop timers - call.hangup(CallErrorCode.UserHangup, true); }); it("should map SDPStreamMetadata to feeds", async () => { @@ -734,16 +733,18 @@ describe('Call', function() { describe("ignoring streams with ids for which we already have a feed", () => { const STREAM_ID = "stream_id"; - const FEEDS_CHANGED_CALLBACK = jest.fn(); + let FEEDS_CHANGED_CALLBACK: jest.Mock; beforeEach(async () => { + FEEDS_CHANGED_CALLBACK = jest.fn(); + await startVoiceCall(client, call); call.on(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); jest.spyOn(call, "pushLocalFeed"); }); afterEach(() => { - FEEDS_CHANGED_CALLBACK.mockReset(); + call.off(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); }); it("should ignore stream passed to pushRemoteFeed()", async () => { @@ -941,4 +942,70 @@ describe('Call', function() { expect(call.state).toEqual(CallState.Ended); }); + + describe("Screen sharing", () => { + beforeEach(async () => { + await startVoiceCall(client, call); + + await call.onAnswerReceived({ + getContent: () => { + return { + "version": 1, + "call_id": call.callId, + "party_id": 'party_id', + "answer": { + sdp: DUMMY_SDP, + }, + "org.matrix.msc3077.sdp_stream_metadata": { + "foo": { + "purpose": "m.usermedia", + "audio_muted": false, + "video_muted": false, + }, + }, + }; + }, + getSender: () => "@test:foo", + }); + }); + + afterEach(() => { + // Hangup to stop timers + call.hangup(CallErrorCode.UserHangup, true); + }); + + it("enables and disables screensharing", async () => { + await call.setScreensharingEnabled(true); + + expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(1); + + client.client.sendEvent.mockReset(); + const sendNegotiatePromise = new Promise(resolve => { + client.client.sendEvent.mockImplementationOnce(() => { + resolve(); + }); + }); + + MockRTCPeerConnection.triggerAllNegotiations(); + await sendNegotiatePromise; + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallNegotiate, + expect.objectContaining({ + "version": "1", + "call_id": call.callId, + "org.matrix.msc3077.sdp_stream_metadata": expect.objectContaining({ + [SCREENSHARE_STREAM_ID]: expect.objectContaining({ + purpose: SDPStreamMetadataPurpose.Screenshare, + }), + }), + }), + ); + + await call.setScreensharingEnabled(false); + + expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(0); + }); + }); }); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 8e14e06a880..4889c942110 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2240,7 +2240,7 @@ export class MatrixCall extends TypedEventEmitter Date: Fri, 26 Aug 2022 10:04:07 +0100 Subject: [PATCH 239/291] Fix ICE end-of-candidates messages (#2622) * Fix ICE end-of-candidates messages We were casting a POJO to an RTCIceCandidate for the dummy end-of-candidates candidate, but https://github.com/matrix-org/matrix-js-sdk/pull/2473 started calling .toJSON() on these objects. Store separately whether we've seen the end of candidates rather than adding on a dummy candidate object. A test for this will follow, but a) I want to get this fix out and b) I'm currently rewriting the call test file to add typing. Fixes https://github.com/vector-im/element-call/issues/553 * Remove hacks for testing * Switch if branches --- src/webrtc/call.ts | 47 +++++++++++++++++++++++++++------------------- 1 file changed, 28 insertions(+), 19 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4889c942110..4b5dcbae614 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -343,7 +343,7 @@ export class MatrixCall extends TypedEventEmitter = []; private candidateSendTries = 0; - private sentEndOfCandidates = false; + private candidatesEnded = false; private feeds: Array = []; private usermediaSenders: Array = []; private screensharingSenders: Array = []; @@ -1597,6 +1597,11 @@ export class MatrixCall extends TypedEventEmitter => { if (event.candidate) { + if (this.candidatesEnded) { + logger.warn("Got candidate after candidates have ended - ignoring!"); + return; + } + logger.debug( "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + event.candidate.candidate, @@ -1606,29 +1611,18 @@ export class MatrixCall extends TypedEventEmitter { logger.debug(`Call ${this.callId} ice gathering state changed to ${this.peerConn.iceGatheringState}`); - if (this.peerConn.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; + if (this.peerConn.iceGatheringState === 'complete') { + this.queueCandidate(null); } }; @@ -2247,7 +2241,12 @@ export class MatrixCall extends TypedEventEmitter candidate.toJSON()) }; + if (this.candidatesEnded) { + // If there are no more candidates, signal this by adding an empty string candidate + content.candidates.push({ + candidate: '', + }); + } logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { await this.sendVoipEvent(EventType.CallCandidates, content); From d5b82e343a99f001fc6c4d3a64481a07cfc92702 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 31 Aug 2022 11:15:13 +0100 Subject: [PATCH 240/291] Add types to the call unit test suites (#2627) * Add types to the call unit test suites Still involves quite a few casts to any unfortunately as it turns out we access quite a few private methods on the Call class in these tests. * Remove commented line & use better expect syntax * Replace more calls.length with toHaveBeenCalled * Remove mistakenly added id field --- spec/test-utils/webrtc.ts | 5 + spec/unit/webrtc/call.spec.ts | 577 ++++++++++++++++------------------ 2 files changed, 269 insertions(+), 313 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index e2118efa466..d8632e09bfe 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -212,6 +212,11 @@ export class MockMediaStream { clone() { return new MockMediaStream(this.id, this.tracks); } + + // syntactic sugar for typing + typed(): MediaStream { + return this as unknown as MediaStream; + } } export class MockMediaDeviceInfo { diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index cac83f4a832..55926e28e53 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -34,7 +34,7 @@ import { SCREENSHARE_STREAM_ID, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; -import { EventType, MatrixEvent } from "../../../src"; +import { Callback, EventType, IContent, MatrixEvent, Room } from "../../../src"; const FAKE_ROOM_ID = "!foo:bar"; const CALL_LIFETIME = 60000; @@ -80,12 +80,23 @@ const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: s await callPromise; }; +function makeMockEvent(sender: string, content: Record): MatrixEvent { + return { + getContent: () => { + return content; + }, + getSender: () => sender, + } as MatrixEvent; +} + describe('Call', function() { - let client; - let call; - let prevNavigator; - let prevDocument; - let prevWindow; + let client: TestClient; + let call: MatrixCall; + let prevNavigator: Navigator; + let prevDocument: Document; + let prevWindow: Window & typeof globalThis; + // We retain a reference to this in the correct Mock type + let mockSendEvent: jest.Mock]>; beforeEach(function() { prevNavigator = global.navigator; @@ -97,17 +108,21 @@ describe('Call', function() { client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); // We just stub out sendEvent: we're not interested in testing the client's // event sending code here - client.client.sendEvent = jest.fn(); - client.client.mediaHandler = new MockMediaHandler; - client.client.getMediaHandler = () => client.client.mediaHandler; - client.client.turnServersExpiry = Date.now() + 60 * 60 * 1000; + client.client.sendEvent = mockSendEvent = jest.fn(); + { + // in which we do naughty assignments to private members + const untypedClient = (client.client as any); + untypedClient.mediaHandler = new MockMediaHandler; + untypedClient.turnServersExpiry = Date.now() + 60 * 60 * 1000; + } + client.httpBackend.when("GET", "/voip/turnServer").respond(200, {}); client.client.getRoom = () => { return { getMember: () => { return {}; }, - }; + } as unknown as Room; }; call = new MatrixCall({ @@ -115,7 +130,7 @@ describe('Call', function() { roomId: FAKE_ROOM_ID, }); // call checks one of these is wired up - call.on('error', () => {}); + call.on(CallEvent.Error, () => {}); }); afterEach(function() { @@ -133,114 +148,84 @@ describe('Call', function() { it('should ignore candidate events from non-matching party ID', async function() { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; - }, - getSender: () => "@test:foo", - }); - - call.peerConn.addIceCandidate = jest.fn(); - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - candidates: [ - { - candidate: '', - sdpMid: '', - }, - ], - }; - }, - getSender: () => "@test:foo", - }); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); - - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'some_other_party_id', - candidates: [ - { - candidate: '', - sdpMid: '', - }, - ], - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); + })); + + const mockAddIceCandidate = call.peerConn.addIceCandidate = jest.fn(); + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + candidates: [ + { + candidate: '', + sdpMid: '', + }, + ], + })); + expect(mockAddIceCandidate).toHaveBeenCalled(); + + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'some_other_party_id', + candidates: [ + { + candidate: '', + sdpMid: '', + }, + ], + })); + expect(mockAddIceCandidate).toHaveBeenCalled(); }); it('should add candidates received before answer if party ID is correct', async function() { await startVoiceCall(client, call); - call.peerConn.addIceCandidate = jest.fn(); - - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - candidates: [ - { - candidate: 'the_correct_candidate', - sdpMid: '', - }, - ], - }; - }, - getSender: () => "@test:foo", - }); - - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'some_other_party_id', - candidates: [ - { - candidate: 'the_wrong_candidate', - sdpMid: '', - }, - ], - }; - }, - getSender: () => "@test:foo", - }); + const mockAddIceCandidate = call.peerConn.addIceCandidate = jest.fn(); + + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + candidates: [ + { + candidate: 'the_correct_candidate', + sdpMid: '', + }, + ], + })); + + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'some_other_party_id', + candidates: [ + { + candidate: 'the_wrong_candidate', + sdpMid: '', + }, + ], + })); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(0); + expect(mockAddIceCandidate).not.toHaveBeenCalled(); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + })); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); - expect(call.peerConn.addIceCandidate).toHaveBeenCalledWith({ + expect(mockAddIceCandidate).toHaveBeenCalled(); + expect(mockAddIceCandidate).toHaveBeenCalledWith({ candidate: 'the_correct_candidate', sdpMid: '', }); @@ -248,37 +233,27 @@ describe('Call', function() { it('should map asserted identity messages to remoteAssertedIdentity', async function() { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + })); const identChangedCallback = jest.fn(); call.on(CallEvent.AssertedIdentityChanged, identChangedCallback); - await call.onAssertedIdentityReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - asserted_identity: { - id: "@steve:example.com", - display_name: "Steve Gibbons", - }, - }; + await call.onAssertedIdentityReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + asserted_identity: { + id: "@steve:example.com", + display_name: "Steve Gibbons", }, - getSender: () => "@test:foo", - }); + })); expect(identChangedCallback).toHaveBeenCalled(); @@ -290,28 +265,23 @@ describe('Call', function() { it("should map SDPStreamMetadata to feeds", async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: { - "remote_stream": { - purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: true, - video_muted: false, - }, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + [SDPStreamMetadataKey]: { + "remote_stream": { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, + }, + }, + })); - call.pushRemoteFeed( + (call as any).pushRemoteFeed( new MockMediaStream( "remote_stream", [ @@ -329,40 +299,35 @@ describe('Call', function() { it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + })); - call.setScreensharingEnabledWithoutMetadataSupport = jest.fn(); + const mockScreenshareNoMetadata = (call as any).setScreensharingEnabledWithoutMetadataSupport = jest.fn(); call.setScreensharingEnabled(true); - expect(call.setScreensharingEnabledWithoutMetadataSupport).toHaveBeenCalled(); + expect(mockScreenshareNoMetadata).toHaveBeenCalled(); }); it("should fallback to answering with no video", async () => { - await client.httpBackend.flush(); + await client.httpBackend.flush(""); - call.shouldAnswerWithMediaType = (wantedValue: boolean) => wantedValue; - client.client.mediaHandler.getUserMediaStream = jest.fn().mockRejectedValue("reject"); + (call as any).shouldAnswerWithMediaType = (wantedValue: boolean) => wantedValue; + client.client.getMediaHandler().getUserMediaStream = jest.fn().mockRejectedValue("reject"); await call.answer(true, true); - expect(client.client.mediaHandler.getUserMediaStream).toHaveBeenNthCalledWith(1, true, true); - expect(client.client.mediaHandler.getUserMediaStream).toHaveBeenNthCalledWith(2, true, false); + expect(client.client.getMediaHandler().getUserMediaStream).toHaveBeenNthCalledWith(1, true, true); + expect(client.client.getMediaHandler().getUserMediaStream).toHaveBeenNthCalledWith(2, true, false); }); it("should handle mid-call device changes", async () => { - client.client.mediaHandler.getUserMediaStream = jest.fn().mockReturnValue( + client.client.getMediaHandler().getUserMediaStream = jest.fn().mockReturnValue( new MockMediaStream( "stream", [ new MockMediaStreamTrack("audio_track", "audio"), @@ -373,19 +338,14 @@ describe('Call', function() { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + })); await call.updateLocalUsermediaStream( new MockMediaStream( @@ -394,15 +354,18 @@ describe('Call', function() { new MockMediaStreamTrack("new_audio_track", "audio"), new MockMediaStreamTrack("video_track", "video"), ], - ), + ).typed(), ); + + const usermediaSenders: Array = (call as any).usermediaSenders; + expect(call.localUsermediaStream.id).toBe("stream"); expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("new_audio_track"); expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); - expect(call.usermediaSenders.find((sender) => { + expect(usermediaSenders.find((sender) => { return sender?.track?.kind === "audio"; }).track.id).toBe("new_audio_track"); - expect(call.usermediaSenders.find((sender) => { + expect(usermediaSenders.find((sender) => { return sender?.track?.kind === "video"; }).track.id).toBe("video_track"); }); @@ -410,29 +373,28 @@ describe('Call', function() { it("should handle upgrade to video call", async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: {}, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + [SDPStreamMetadataKey]: {}, + })); - await call.upgradeCall(false, true); + // XXX Should probably test using the public interfaces, ie. + // setLocalVideoMuted probably? + await (call as any).upgradeCall(false, true); + + const usermediaSenders: Array = (call as any).usermediaSenders; expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("audio_track"); expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); - expect(call.usermediaSenders.find((sender) => { + expect(usermediaSenders.find((sender) => { return sender?.track?.kind === "audio"; }).track.id).toBe("audio_track"); - expect(call.usermediaSenders.find((sender) => { + expect(usermediaSenders.find((sender) => { return sender?.track?.kind === "video"; }).track.id).toBe("video_track"); }); @@ -440,37 +402,35 @@ describe('Call', function() { it("should handle SDPStreamMetadata changes", async () => { await startVoiceCall(client, call); - call.updateRemoteSDPStreamMetadata({ + (call as any).updateRemoteSDPStreamMetadata({ "remote_stream": { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, video_muted: false, }, }); - call.pushRemoteFeed(new MockMediaStream("remote_stream", [])); + (call as any).pushRemoteFeed(new MockMediaStream("remote_stream", [])); const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); - call.onSDPStreamMetadataChangedReceived({ - getContent: () => ({ - [SDPStreamMetadataKey]: { - "remote_stream": { - purpose: SDPStreamMetadataPurpose.Screenshare, - audio_muted: true, - video_muted: true, - id: "feed_id2", - }, + call.onSDPStreamMetadataChangedReceived(makeMockEvent("@test:foo", { + [SDPStreamMetadataKey]: { + "remote_stream": { + purpose: SDPStreamMetadataPurpose.Screenshare, + audio_muted: true, + video_muted: true, + id: "feed_id2", }, - }), - }); + }, + })); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Screenshare); - expect(feed?.audioMuted).toBe(true); - expect(feed?.videoMuted).toBe(true); + expect(feed?.isAudioMuted()).toBe(true); + expect(feed?.isVideoMuted()).toBe(true); }); it("should choose opponent member", async () => { const callPromise = call.placeVoiceCall(); - await client.httpBackend.flush(); + await client.httpBackend.flush(""); await callPromise; const opponentMember = { @@ -485,25 +445,22 @@ describe('Call', function() { return opponentMember; } }, - }; + } as unknown as Room; }; const opponentCaps = { "m.call.transferee": true, "m.call.dtmf": false, }; - call.chooseOpponent({ - getContent: () => ({ - version: 1, - party_id: "party_id", - capabilities: opponentCaps, - }), - getSender: () => opponentMember.userId, - }); + (call as any).chooseOpponent(makeMockEvent(opponentMember.userId, { + version: 1, + party_id: "party_id", + capabilities: opponentCaps, + })); expect(call.getOpponentMember()).toBe(opponentMember); - expect(call.opponentPartyId).toBe("party_id"); - expect(call.opponentCaps).toBe(opponentCaps); + expect((call as any).opponentPartyId).toBe("party_id"); + expect((call as any).opponentCaps).toBe(opponentCaps); expect(call.opponentCanBeTransferred()).toBe(true); expect(call.opponentSupportsDTMF()).toBe(false); }); @@ -512,21 +469,23 @@ describe('Call', function() { it("if no video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + (call as any).pushRemoteFeed(new MockMediaStream("remote_stream1", [])); expect(call.type).toBe(CallType.Voice); }); it("if remote video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")])); + (call as any).pushRemoteFeed( + new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), + ); expect(call.type).toBe(CallType.Video); }); it("if local video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushNewLocalFeed( + (call as any).pushNewLocalFeed( new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), SDPStreamMetadataPurpose.Usermedia, false, @@ -537,7 +496,7 @@ describe('Call', function() { it("should correctly generate local SDPStreamMetadata", async () => { const callPromise = call.placeCallWithCallFeeds([new CallFeed({ - client, + client: client.client, // @ts-ignore Mock stream: new MockMediaStream("local_stream1", [new MockMediaStreamTrack("track_id", "audio")]), roomId: call.roomId, @@ -546,17 +505,17 @@ describe('Call', function() { audioMuted: false, videoMuted: false, })]); - await client.httpBackend.flush(); + await client.httpBackend.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushNewLocalFeed( + (call as any).pushNewLocalFeed( new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]), SDPStreamMetadataPurpose.Screenshare, "feed_id2", ); await call.setMicrophoneMuted(true); - expect(call.getLocalSDPStreamMetadata()).toStrictEqual({ + expect((call as any).getLocalSDPStreamMetadata()).toStrictEqual({ "local_stream1": { "purpose": SDPStreamMetadataPurpose.Usermedia, "audio_muted": true, @@ -578,7 +537,7 @@ describe('Call', function() { const callPromise = call.placeCallWithCallFeeds([ new CallFeed({ - client, + client: client.client, userId: client.getUserId(), // @ts-ignore Mock stream: localUsermediaStream, @@ -588,7 +547,7 @@ describe('Call', function() { videoMuted: false, }), new CallFeed({ - client, + client: client.client, userId: client.getUserId(), // @ts-ignore Mock stream: localScreensharingStream, @@ -598,14 +557,13 @@ describe('Call', function() { videoMuted: false, }), ]); - await client.httpBackend.flush(); + await client.httpBackend.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.updateRemoteSDPStreamMetadata({ + (call as any).updateRemoteSDPStreamMetadata({ "remote_usermedia_stream_id": { purpose: SDPStreamMetadataPurpose.Usermedia, - id: "remote_usermedia_feed_id", audio_muted: false, video_muted: false, }, @@ -616,8 +574,8 @@ describe('Call', function() { video_muted: false, }, }); - call.pushRemoteFeed(remoteUsermediaStream); - call.pushRemoteFeed(remoteScreensharingStream); + (call as any).pushRemoteFeed(remoteUsermediaStream); + (call as any).pushRemoteFeed(remoteScreensharingStream); expect(call.localUsermediaFeed.stream).toBe(localUsermediaStream); expect(call.localUsermediaStream).toBe(localUsermediaStream); @@ -636,14 +594,12 @@ describe('Call', function() { const callHangupCallback = jest.fn(); call.on(CallEvent.Hangup, callHangupCallback); - await call.onSelectAnswerReceived({ - getContent: () => ({ - version: 1, - call_id: call.callId, - party_id: 'party_id', - selected_party_id: "different_party_id", - }), - }); + await call.onSelectAnswerReceived(makeMockEvent("@test:foo.bar", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + selected_party_id: "different_party_id", + })); expect(callHangupCallback).toHaveBeenCalled(); }); @@ -748,43 +704,38 @@ describe('Call', function() { }); it("should ignore stream passed to pushRemoteFeed()", async () => { - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: { - [STREAM_ID]: { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + [SDPStreamMetadataKey]: { + [STREAM_ID]: { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }, + })); - call.pushRemoteFeed(new MockMediaStream(STREAM_ID)); - call.pushRemoteFeed(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); it("should ignore stream passed to pushRemoteFeedWithoutMetadata()", async () => { - call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); - call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); it("should ignore stream passed to pushNewLocalFeed()", async () => { - call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); - call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); // We already have one local feed from placeVoiceCall() expect(call.getLocalFeeds().length).toBe(2); @@ -794,15 +745,16 @@ describe('Call', function() { }); describe("muting", () => { + let mockSendVoipEvent: jest.Mock, [string, object]>; beforeEach(async () => { - call.sendVoipEvent = jest.fn(); + (call as any).sendVoipEvent = mockSendVoipEvent = jest.fn(); await startVideoCall(client, call); }); describe("sending sdp_stream_metadata_changed events", () => { it("should send sdp_stream_metadata_changed when muting audio", async () => { await call.setMicrophoneMuted(true); - expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: { mock_stream_from_media_handler: { purpose: SDPStreamMetadataPurpose.Usermedia, @@ -815,7 +767,7 @@ describe('Call', function() { it("should send sdp_stream_metadata_changed when muting video", async () => { await call.setLocalVideoMuted(true); - expect(call.sendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { [SDPStreamMetadataKey]: { mock_stream_from_media_handler: { purpose: SDPStreamMetadataPurpose.Usermedia, @@ -836,7 +788,7 @@ describe('Call', function() { video_muted: video, }, }; - call.pushRemoteFeed(new MockMediaStream("stream", [ + (call as any).pushRemoteFeed(new MockMediaStream("stream", [ new MockMediaStreamTrack("track1", "audio"), new MockMediaStreamTrack("track1", "video"), ])); @@ -844,20 +796,20 @@ describe('Call', function() { getContent: () => ({ [SDPStreamMetadataKey]: metadata, }), - }); + } as MatrixEvent); return metadata; }; it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { const metadata = setupCall(true, false); - expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata); + expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true); expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false); }); it("should handle incoming sdp_stream_metadata_changed with video muted", async () => { const metadata = setupCall(false, true); - expect(call.remoteSDPStreamMetadata).toStrictEqual(metadata); + expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false); expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true); }); @@ -898,7 +850,7 @@ describe('Call', function() { await call.answer(); - client.client.sendEvent.mockReset(); + mockSendEvent.mockReset(); let caught = false; try { @@ -910,7 +862,7 @@ describe('Call', function() { expect(caught).toEqual(true); expect(client.client.sendEvent).not.toHaveBeenCalled(); - call.hangup(); + call.hangup(CallErrorCode.UserHangup, true); }); it("hangs up a call", async () => { @@ -918,9 +870,9 @@ describe('Call', function() { await call.answer(); - client.client.sendEvent.mockReset(); + mockSendEvent.mockReset(); - call.hangup(); + call.hangup(CallErrorCode.UserHangup, true); expect(client.client.sendEvent).toHaveBeenCalledWith( FAKE_ROOM_ID, @@ -947,26 +899,21 @@ describe('Call', function() { beforeEach(async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - "version": 1, - "call_id": call.callId, - "party_id": 'party_id', - "answer": { - sdp: DUMMY_SDP, - }, - "org.matrix.msc3077.sdp_stream_metadata": { - "foo": { - "purpose": "m.usermedia", - "audio_muted": false, - "video_muted": false, - }, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + "version": 1, + "call_id": call.callId, + "party_id": 'party_id', + "answer": { + sdp: DUMMY_SDP, }, - getSender: () => "@test:foo", - }); + "org.matrix.msc3077.sdp_stream_metadata": { + "foo": { + "purpose": "m.usermedia", + "audio_muted": false, + "video_muted": false, + }, + }, + })); }); afterEach(() => { @@ -977,11 +924,13 @@ describe('Call', function() { it("enables and disables screensharing", async () => { await call.setScreensharingEnabled(true); - expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(1); + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length, + ).toEqual(1); - client.client.sendEvent.mockReset(); + mockSendEvent.mockReset(); const sendNegotiatePromise = new Promise(resolve => { - client.client.sendEvent.mockImplementationOnce(() => { + mockSendEvent.mockImplementationOnce(() => { resolve(); }); }); @@ -1005,7 +954,9 @@ describe('Call', function() { await call.setScreensharingEnabled(false); - expect(call.feeds.filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length).toEqual(0); + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length, + ).toEqual(0); }); }); }); From db32420d16285bf4991729f4801e0b247b16e2fb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 31 Aug 2022 13:40:55 +0100 Subject: [PATCH 241/291] Add logging to diagnose connection issue (#2629) For https://github.com/vector-im/element-call/issues/559 --- src/webrtc/groupCall.ts | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 6672dbd539b..3cb24c9bed0 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -748,11 +748,21 @@ export class GroupCall extends TypedEventEmitter< } public onMemberStateChanged = async (event: MatrixEvent) => { + // If we haven't entered the call yet, we don't care + if (this.state !== GroupCallState.Entered) { + return; + } + // The member events may be received for another room, which we will ignore. if (event.getRoomId() !== this.room.roomId) return; const member = this.room.getMember(event.getStateKey()); - if (!member) return; + if (!member) { + logger.warn(`Couldn't find room member for ${event.getStateKey()}: ignoring member state event!`); + return; + } + + logger.debug(`Processing member state event for ${member.userId}`); const ignore = () => { this.removeParticipant(member); @@ -766,7 +776,7 @@ export class GroupCall extends TypedEventEmitter< : []; // Ignore expired device data if (callsState.length === 0) { - logger.log(`Ignoring member state from ${member.userId} member not in any calls.`); + logger.info(`Ignoring member state from ${member.userId} member not in any calls.`); ignore(); return; } @@ -802,14 +812,10 @@ export class GroupCall extends TypedEventEmitter< return; } - if (this.state !== GroupCallState.Entered) { - return; - } - // Only initiate a call with a user who has a userId that is lexicographically // less than your own. Otherwise, that user will call you. if (member.userId < localUserId) { - logger.log(`Waiting for ${member.userId} to send call invite.`); + logger.debug(`Waiting for ${member.userId} to send call invite.`); return; } @@ -849,6 +855,8 @@ export class GroupCall extends TypedEventEmitter< const requestScreenshareFeed = opponentDevice.feeds.some( (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + logger.log(`Placing call to ${member.userId}.`); + try { await newCall.placeCallWithCallFeeds( this.getLocalFeeds().map(feed => feed.clone()), From 0981652de4c2ab96587cf15d118279034addc823 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 1 Sep 2022 16:18:37 +0100 Subject: [PATCH 242/291] Add GroupCallEventHandlerEvent.Room (#2631) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add GroupCalEventHandlerEvent.Room Emit an event when the group call event handler has processed all pending group calls. * Remove unused return value * Add void return type Co-authored-by: Šimon Brandner Co-authored-by: Šimon Brandner --- src/client.ts | 1 + src/webrtc/groupCallEventHandler.ts | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/client.ts b/src/client.ts index 3ec0dcb8590..472105222ab 100644 --- a/src/client.ts +++ b/src/client.ts @@ -869,6 +869,7 @@ type EmittedEvents = ClientEvent | GroupCallEventHandlerEvent.Incoming | GroupCallEventHandlerEvent.Ended | GroupCallEventHandlerEvent.Participants + | GroupCallEventHandlerEvent.Room | HttpApiEvent.SessionLoggedOut | HttpApiEvent.NoConsent | BeaconEvent; diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 9d7f2e5afd4..94a81f249ce 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -33,12 +33,14 @@ export enum GroupCallEventHandlerEvent { Incoming = "GroupCall.incoming", Ended = "GroupCall.ended", Participants = "GroupCall.participants", + Room = "GroupCall.Room", } export type GroupCallEventHandlerEventHandlerMap = { [GroupCallEventHandlerEvent.Incoming]: (call: GroupCall) => void; [GroupCallEventHandlerEvent.Ended]: (call: GroupCall) => void; [GroupCallEventHandlerEvent.Participants]: (participants: RoomMember[], call: GroupCall) => void; + [GroupCallEventHandlerEvent.Room]: (room: Room) => void; }; export class GroupCallEventHandler { @@ -65,7 +67,7 @@ export class GroupCallEventHandler { return [...this.groupCalls.values()].find((groupCall) => groupCall.groupCallId === groupCallId); } - private createGroupCallForRoom(room: Room): GroupCall | undefined { + private createGroupCallForRoom(room: Room): void { const callEvents = room.currentState.getStateEvents(EventType.GroupCallPrefix); const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); @@ -76,7 +78,9 @@ export class GroupCallEventHandler { continue; } - return this.createGroupCallFromRoomStateEvent(callEvent); + this.createGroupCallFromRoomStateEvent(callEvent); + this.client.emit(GroupCallEventHandlerEvent.Room, room); + break; } } From d656b848f85beb11c3672c9473697cf67985b601 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 1 Sep 2022 17:57:38 +0100 Subject: [PATCH 243/291] Wait for client to start syncing before making group calls (#2632) As hopefully explained in comment Fixes https://github.com/matrix-org/matrix-js-sdk/issues/2589 --- src/webrtc/groupCallEventHandler.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 94a81f249ce..0a4b99223be 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -28,6 +28,7 @@ import { RoomState } from "../models/room-state"; import { RoomMember } from "../models/room-member"; import { logger } from '../logger'; import { EventType } from "../@types/event"; +import { SyncState } from '../sync'; export enum GroupCallEventHandlerEvent { Incoming = "GroupCall.incoming", @@ -48,7 +49,25 @@ export class GroupCallEventHandler { constructor(private client: MatrixClient) { } - public start(): void { + public async start(): Promise { + // We wait until the client has started syncing for real. + // This is because we only support one call at a time, and want + // the latest. We therefore want the latest state of the room before + // we create a group call for the room so we can be fairly sure that + // the group call we create is really the latest one. + if (this.client.getSyncState() !== SyncState.Syncing) { + logger.debug("Waiting for client to start syncing..."); + await new Promise(resolve => { + const onSync = () => { + if (this.client.getSyncState() === SyncState.Syncing) { + this.client.off(ClientEvent.Sync, onSync); + return resolve(); + } + }; + this.client.on(ClientEvent.Sync, onSync); + }); + } + const rooms = this.client.getRooms(); for (const room of rooms) { From 40ecfa793263bf764fc0c6b2d45bbf911af7717f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 2 Sep 2022 12:57:29 +0100 Subject: [PATCH 244/291] Test disabling screenshare in group calls (#2634) Also add a few more types --- spec/test-utils/webrtc.ts | 8 ++++--- spec/unit/webrtc/groupCall.spec.ts | 37 ++++++++++++++++++++++-------- 2 files changed, 32 insertions(+), 13 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index d8632e09bfe..b82137bbe4e 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { IScreensharingOpts } from "../../src/webrtc/mediaHandler"; +import { IScreensharingOpts, MediaHandler } from "../../src/webrtc/mediaHandler"; export const DUMMY_SDP = ( "v=0\r\n" + @@ -241,20 +241,22 @@ export class MockMediaHandler { stopUserMediaStream(stream: MockMediaStream) { stream.isStopped = true; } - getScreensharingStream(opts?: IScreensharingOpts) { + getScreensharingStream = jest.fn((opts?: IScreensharingOpts) => { const tracks = [new MockMediaStreamTrack("video_track", "video")]; if (opts?.audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); const stream = new MockMediaStream(SCREENSHARE_STREAM_ID, tracks); this.screensharingStreams.push(stream); return stream; - } + }); stopScreensharingStream(stream: MockMediaStream) { stream.isStopped = true; } hasAudioDevice() { return true; } hasVideoDevice() { return true; } stopAllStreams() {} + + typed(): MediaHandler { return this as unknown as MediaHandler; } } export function installWebRTCMocks() { diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 0423930ae2f..861b45ad080 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -98,35 +98,37 @@ const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise { - public mediaHandler: MediaHandler = new MockMediaHandler() as unknown as MediaHandler; + public mediaHandler = new MockMediaHandler(); constructor(public userId: string, public deviceId: string, public sessionId: string) { super(); } groupCallEventHandler = { - groupCalls: new Map(), + groupCalls: new Map(), }; callEventHandler = { - calls: new Map(), + calls: new Map(), }; sendStateEvent = jest.fn(); sendToDevice = jest.fn(); - getMediaHandler() { return this.mediaHandler; } + getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } - getUserId() { return this.userId; } + getUserId(): string { return this.userId; } - getDeviceId() { return this.deviceId; } - getSessionId() { return this.sessionId; } + getDeviceId(): string { return this.deviceId; } + getSessionId(): string { return this.sessionId; } getTurnServers = () => []; isFallbackICEServerAllowed = () => false; reEmitter = new ReEmitter(new TypedEventEmitter()); getUseE2eForGroupCall = () => false; checkTurnServers = () => null; + + typed(): MatrixClient { return this as unknown as MatrixClient; } } class MockCall { @@ -732,15 +734,16 @@ describe('Group Call', function() { }); describe("screensharing", () => { + let typedMockClient: MockCallMatrixClient; let mockClient: MatrixClient; let room: Room; let groupCall: GroupCall; beforeEach(async () => { - const typedMockClient = new MockCallMatrixClient( + typedMockClient = new MockCallMatrixClient( FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, ); - mockClient = typedMockClient as unknown as MatrixClient; + mockClient = typedMockClient.typed(); room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); @@ -761,7 +764,10 @@ describe('Group Call', function() { return call.gotLocalOffer; }); - await groupCall.setScreensharingEnabled(true); + let enabledResult; + enabledResult = await groupCall.setScreensharingEnabled(true); + expect(enabledResult).toEqual(true); + expect(typedMockClient.mediaHandler.getScreensharingStream).toHaveBeenCalled(); MockRTCPeerConnection.triggerAllNegotiations(); expect(groupCall.screenshareFeeds).toHaveLength(1); @@ -770,6 +776,17 @@ describe('Group Call', function() { }); onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled()); + // Enabling it again should do nothing + typedMockClient.mediaHandler.getScreensharingStream.mockClear(); + enabledResult = await groupCall.setScreensharingEnabled(true); + expect(enabledResult).toEqual(true); + expect(typedMockClient.mediaHandler.getScreensharingStream).not.toHaveBeenCalled(); + + // Should now be able to disable it + enabledResult = await groupCall.setScreensharingEnabled(false); + expect(enabledResult).toEqual(false); + expect(groupCall.screenshareFeeds).toHaveLength(0); + groupCall.terminate(); }); From 0d6a93b5f6002997321856d409337cdcaf1d0cc8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 2 Sep 2022 15:33:22 +0100 Subject: [PATCH 245/291] Refactor the group call placing calls test (#2636) Add some types & use mock-typed versions directly - it's clearer which client we're making assertions about. --- spec/unit/webrtc/groupCall.spec.ts | 61 ++++++++++++++++++------------ 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 861b45ad080..86758ff892c 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -14,7 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { EventType, GroupCallIntent, GroupCallType, MatrixCall, MatrixEvent, Room, RoomMember } from '../../../src'; +import { + EventType, + GroupCallIntent, + GroupCallType, + ISendEventResponse, + MatrixCall, + MatrixEvent, + Room, + RoomMember, +} from '../../../src'; import { GroupCall } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; import { @@ -112,8 +121,14 @@ class MockCallMatrixClient extends TypedEventEmitter(), }; - sendStateEvent = jest.fn(); - sendToDevice = jest.fn(); + sendStateEvent = jest.fn, [ + roomId: string, eventType: EventType, content: any, statekey: string, + ]>(); + sendToDevice = jest.fn, [ + eventType: string, + contentMap: { [userId: string]: { [deviceId: string]: Record } }, + txnId?: string, + ]>(); getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } @@ -407,21 +422,24 @@ describe('Group Call', function() { describe('Placing calls', function() { let groupCall1: GroupCall; let groupCall2: GroupCall; - let client1: MatrixClient; - let client2: MatrixClient; + let client1: MockCallMatrixClient; + let client2: MockCallMatrixClient; beforeEach(function() { MockRTCPeerConnection.resetInstances(); client1 = new MockCallMatrixClient( FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, - ) as unknown as MatrixClient; + ); client2 = new MockCallMatrixClient( FAKE_USER_ID_2, FAKE_DEVICE_ID_2, FAKE_SESSION_ID_2, - ) as unknown as MatrixClient; + ); - client1.sendStateEvent = client2.sendStateEvent = (roomId, eventType, content, statekey) => { + // Inject the state events directly into each client when sent + const fakeSendStateEvents = ( + roomId: string, eventType: EventType, content: any, statekey: string, + ) => { if (eventType === EventType.GroupCallMemberPrefix) { const fakeEvent = { getContent: () => content, @@ -445,16 +463,19 @@ describe('Group Call', function() { return Promise.resolve(null); }; - const client1Room = new Room(FAKE_ROOM_ID, client1, FAKE_USER_ID_1); + client1.sendStateEvent.mockImplementation(fakeSendStateEvents); + client2.sendStateEvent.mockImplementation(fakeSendStateEvents); - const client2Room = new Room(FAKE_ROOM_ID, client2, FAKE_USER_ID_2); + const client1Room = new Room(FAKE_ROOM_ID, client1.typed(), FAKE_USER_ID_1); + + const client2Room = new Room(FAKE_ROOM_ID, client2.typed(), FAKE_USER_ID_2); groupCall1 = new GroupCall( - client1, client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, + client1.typed(), client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, ); groupCall2 = new GroupCall( - client2, client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, + client2.typed(), client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, ); client1Room.currentState.members[FAKE_USER_ID_1] = { @@ -480,32 +501,22 @@ describe('Group Call', function() { await groupCall1.create(); try { - // keep this as its own variable so we have it typed as a mock - // rather than its type in the client object - const mockSendToDevice = jest.fn, [ - eventType: string, - contentMap: { [userId: string]: { [deviceId: string]: Record } }, - txnId?: string, - ]>(); - const toDeviceProm = new Promise(resolve => { - mockSendToDevice.mockImplementation(() => { + client1.sendToDevice.mockImplementation(() => { resolve(); return Promise.resolve({}); }); }); - client1.sendToDevice = mockSendToDevice; - await Promise.all([groupCall1.enter(), groupCall2.enter()]); MockRTCPeerConnection.triggerAllNegotiations(); await toDeviceProm; - expect(mockSendToDevice.mock.calls[0][0]).toBe("m.call.invite"); + expect(client1.sendToDevice.mock.calls[0][0]).toBe("m.call.invite"); - const toDeviceCallContent = mockSendToDevice.mock.calls[0][1]; + const toDeviceCallContent = client1.sendToDevice.mock.calls[0][1]; expect(Object.keys(toDeviceCallContent).length).toBe(1); expect(Object.keys(toDeviceCallContent)[0]).toBe(FAKE_USER_ID_2); From c78631bdee3d79d8448c9c92e4763b1533e28e8b Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 5 Sep 2022 09:45:32 +0100 Subject: [PATCH 246/291] Test that calls in a group call are retried (#2637) * Test that calls in a group call are retried * Add new flushpromises file --- spec/test-utils/flushPromises.ts | 28 ++++++++++++ spec/test-utils/webrtc.ts | 8 ++++ spec/unit/queueToDevice.spec.ts | 14 +----- spec/unit/webrtc/groupCall.spec.ts | 70 +++++++++++++++++++++++++++++- 4 files changed, 105 insertions(+), 15 deletions(-) create mode 100644 spec/test-utils/flushPromises.ts diff --git a/spec/test-utils/flushPromises.ts b/spec/test-utils/flushPromises.ts new file mode 100644 index 00000000000..8512410aa9f --- /dev/null +++ b/spec/test-utils/flushPromises.ts @@ -0,0 +1,28 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Jest now uses @sinonjs/fake-timers which exposes tickAsync() and a number of +// other async methods which break the event loop, letting scheduled promise +// callbacks run. Unfortunately, Jest doesn't expose these, so we have to do +// it manually (this is what sinon does under the hood). We do both in a loop +// until the thing we expect happens: hopefully this is the least flakey way +// and avoids assuming anything about the app's behaviour. +const realSetTimeout = setTimeout; +export function flushPromises() { + return new Promise(r => { + realSetTimeout(r, 1); + }); +} diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index b82137bbe4e..4dd7e1633c9 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -79,6 +79,8 @@ export class MockRTCPeerConnection { private negotiationNeededListener: () => void; private needsNegotiation = false; + public readyToNegotiate: Promise; + private onReadyToNegotiate: () => void; localDescription: RTCSessionDescription; signalingState: RTCSignalingState = "stable"; @@ -99,6 +101,10 @@ export class MockRTCPeerConnection { toJSON: function() { }, }; + this.readyToNegotiate = new Promise(resolve => { + this.onReadyToNegotiate = resolve; + }); + MockRTCPeerConnection.instances.push(this); } @@ -128,11 +134,13 @@ export class MockRTCPeerConnection { getStats() { return []; } addTrack(track: MockMediaStreamTrack) { this.needsNegotiation = true; + this.onReadyToNegotiate(); return new MockRTCRtpSender(track); } removeTrack() { this.needsNegotiation = true; + this.onReadyToNegotiate(); } doNegotiation() { diff --git a/spec/unit/queueToDevice.spec.ts b/spec/unit/queueToDevice.spec.ts index a1ae2bcfe93..d4a8b05a69e 100644 --- a/spec/unit/queueToDevice.spec.ts +++ b/spec/unit/queueToDevice.spec.ts @@ -22,6 +22,7 @@ import { MatrixClient } from "../../src/client"; import { ToDeviceBatch } from '../../src/models/ToDeviceMessage'; import { logger } from '../../src/logger'; import { IStore } from '../../src/store'; +import { flushPromises } from '../test-utils/flushPromises'; const FAKE_USER = "@alice:example.org"; const FAKE_DEVICE_ID = "AAAAAAAA"; @@ -47,19 +48,6 @@ enum StoreType { IndexedDB = 'IndexedDB', } -// Jest now uses @sinonjs/fake-timers which exposes tickAsync() and a number of -// other async methods which break the event loop, letting scheduled promise -// callbacks run. Unfortunately, Jest doesn't expose these, so we have to do -// it manually (this is what sinon does under the hood). We do both in a loop -// until the thing we expect happens: hopefully this is the least flakey way -// and avoids assuming anything about the app's behaviour. -const realSetTimeout = setTimeout; -function flushPromises() { - return new Promise(r => { - realSetTimeout(r, 1); - }); -} - async function flushAndRunTimersUntil(cond: () => boolean) { while (!cond()) { await flushPromises(); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 86758ff892c..18ca1b543fa 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -40,7 +40,8 @@ import { TypedEventEmitter } from '../../../src/models/typed-event-emitter'; import { MediaHandler } from '../../../src/webrtc/mediaHandler'; import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from '../../../src/webrtc/callEventHandler'; import { CallFeed } from '../../../src/webrtc/callFeed'; -import { CallState } from '../../../src/webrtc/call'; +import { CallEvent, CallEventHandlerMap, CallState } from '../../../src/webrtc/call'; +import { flushPromises } from '../../test-utils/flushPromises'; const FAKE_ROOM_ID = "!fake:test.dummy"; const FAKE_CONF_ID = "fakegroupcallid"; @@ -106,7 +107,10 @@ const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise { +type EmittedEvents = CallEventHandlerEvent | CallEvent; +type EmittedEventMap = CallEventHandlerEventHandlerMap & CallEventHandlerMap; + +class MockCallMatrixClient extends TypedEventEmitter { public mediaHandler = new MockMediaHandler(); constructor(public userId: string, public deviceId: string, public sessionId: string) { @@ -494,6 +498,8 @@ describe('Group Call', function() { }); afterEach(function() { + jest.useRealTimers(); + MockRTCPeerConnection.resetInstances(); }); @@ -530,6 +536,66 @@ describe('Group Call', function() { await Promise.all([groupCall1.leave(), groupCall2.leave()]); } }); + + it("Retries calls", async function() { + jest.useFakeTimers(); + await groupCall1.create(); + + try { + const toDeviceProm = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + await Promise.all([groupCall1.enter(), groupCall2.enter()]); + + MockRTCPeerConnection.triggerAllNegotiations(); + + await toDeviceProm; + + expect(client1.sendToDevice).toHaveBeenCalled(); + + const oldCall = groupCall1.getCallByUserId(client2.userId); + oldCall.emit(CallEvent.Hangup, oldCall); + + client1.sendToDevice.mockClear(); + + const toDeviceProm2 = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + jest.advanceTimersByTime(groupCall1.retryCallInterval + 500); + + // when we placed the call, we could await on enter which waited for the call to + // be made. We don't have that luxury now, so first have to wait for the call + // to even be created... + let newCall: MatrixCall; + while ( + (newCall = groupCall1.getCallByUserId(client2.userId)) === undefined || + newCall.callId == oldCall.callId + ) { + await flushPromises(); + } + const mockPc = newCall.peerConn as unknown as MockRTCPeerConnection; + + // ...then wait for it to be ready to negotiate + await mockPc.readyToNegotiate; + + MockRTCPeerConnection.triggerAllNegotiations(); + + // ...and then finally we can wait for the invite to be sent + await toDeviceProm2; + + expect(client1.sendToDevice).toHaveBeenCalledWith(EventType.CallInvite, expect.objectContaining({})); + } finally { + await Promise.all([groupCall1.leave(), groupCall2.leave()]); + } + }); }); describe("muting", () => { From aca51fd8a32bbb9b1f6ff0298fa3f0dde80ddf39 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 5 Sep 2022 17:06:49 +0100 Subject: [PATCH 247/291] Test call mute status set on call state chnage (#2638) --- spec/unit/webrtc/groupCall.spec.ts | 35 ++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 18ca1b543fa..dd8825e40b1 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -596,6 +596,41 @@ describe('Group Call', function() { await Promise.all([groupCall1.leave(), groupCall2.leave()]); } }); + + it("Updates call mute status correctly on call state change", async function() { + await groupCall1.create(); + + try { + const toDeviceProm = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + await Promise.all([groupCall1.enter(), groupCall2.enter()]); + + MockRTCPeerConnection.triggerAllNegotiations(); + + await toDeviceProm; + + groupCall1.setMicrophoneMuted(false); + groupCall1.setLocalVideoMuted(false); + + const call = groupCall1.getCallByUserId(client2.userId); + call.isMicrophoneMuted = jest.fn().mockReturnValue(true); + call.setMicrophoneMuted = jest.fn(); + call.isLocalVideoMuted = jest.fn().mockReturnValue(true); + call.setLocalVideoMuted = jest.fn(); + + call.emit(CallEvent.State, CallState.Connected); + + expect(call.setMicrophoneMuted).toHaveBeenCalledWith(false); + expect(call.setLocalVideoMuted).toHaveBeenCalledWith(false); + } finally { + await Promise.all([groupCall1.leave(), groupCall2.leave()]); + } + }); }); describe("muting", () => { From 98d119d6e1d39f1c5b01b36e7fda133e9f12f50c Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 6 Sep 2022 13:54:48 +0100 Subject: [PATCH 248/291] Add client.waitUntilRoomReadyForGroupCalls() (#2641) See comment, although this still feels like a poor solution to the problem. Might be better if the js-sdk processed everything internally before emitting the 'Room' event (or indeed before joinRoom resolved) so the app knows everything is ready when it gets that event. --- src/client.ts | 16 ++++++++++++- src/webrtc/groupCallEventHandler.ts | 37 ++++++++++++++++++++++++++--- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/src/client.ts b/src/client.ts index 472105222ab..6a566ef9b39 100644 --- a/src/client.ts +++ b/src/client.ts @@ -869,7 +869,6 @@ type EmittedEvents = ClientEvent | GroupCallEventHandlerEvent.Incoming | GroupCallEventHandlerEvent.Ended | GroupCallEventHandlerEvent.Participants - | GroupCallEventHandlerEvent.Room | HttpApiEvent.SessionLoggedOut | HttpApiEvent.NoConsent | BeaconEvent; @@ -1594,6 +1593,21 @@ export class MatrixClient extends TypedEventEmitter { + return this.groupCallEventHandler.waitUntilRoomReadyForGroupCalls(roomId); + } + /** * Get an existing group call for the provided room. * @param roomId diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 0a4b99223be..8be5c880fdb 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -34,19 +34,28 @@ export enum GroupCallEventHandlerEvent { Incoming = "GroupCall.incoming", Ended = "GroupCall.ended", Participants = "GroupCall.participants", - Room = "GroupCall.Room", } export type GroupCallEventHandlerEventHandlerMap = { [GroupCallEventHandlerEvent.Incoming]: (call: GroupCall) => void; [GroupCallEventHandlerEvent.Ended]: (call: GroupCall) => void; [GroupCallEventHandlerEvent.Participants]: (participants: RoomMember[], call: GroupCall) => void; - [GroupCallEventHandlerEvent.Room]: (room: Room) => void; }; +interface RoomDeferred { + prom: Promise; + resolve?: () => void; +} + export class GroupCallEventHandler { public groupCalls = new Map(); // roomId -> GroupCall + // All rooms we know about and whether we've seen a 'Room' event + // for them. The promise will be fulfilled once we've processed that + // event which means we're "up to date" on what calls are in a room + // and get + private roomDeferreds = new Map(); + constructor(private client: MatrixClient) { } public async start(): Promise { @@ -82,6 +91,26 @@ export class GroupCallEventHandler { this.client.removeListener(RoomStateEvent.Events, this.onRoomStateChanged); } + private getRoomDeferred(roomId: string): RoomDeferred { + let deferred: RoomDeferred = this.roomDeferreds.get(roomId); + if (deferred === undefined) { + let resolveFunc: () => void; + deferred = { + prom: new Promise(resolve => { + resolveFunc = resolve; + }), + }; + deferred.resolve = resolveFunc; + this.roomDeferreds.set(roomId, deferred); + } + + return deferred; + } + + public waitUntilRoomReadyForGroupCalls(roomId: string): Promise { + return this.getRoomDeferred(roomId).prom; + } + public getGroupCallById(groupCallId: string): GroupCall { return [...this.groupCalls.values()].find((groupCall) => groupCall.groupCallId === groupCallId); } @@ -98,9 +127,11 @@ export class GroupCallEventHandler { } this.createGroupCallFromRoomStateEvent(callEvent); - this.client.emit(GroupCallEventHandlerEvent.Room, room); break; } + + logger.info("Group call event handler processed room", room); + this.getRoomDeferred(room.roomId).resolve(); } private createGroupCallFromRoomStateEvent(event: MatrixEvent): GroupCall | undefined { From fa6f70f70826e9990838649be74e6370ad261cc4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 6 Sep 2022 18:09:34 +0100 Subject: [PATCH 249/291] Log ID instead of object (#2643) as otherwise it recurses and logs the entire client + store --- src/webrtc/groupCallEventHandler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index 8be5c880fdb..eb4cb7723ab 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -130,7 +130,7 @@ export class GroupCallEventHandler { break; } - logger.info("Group call event handler processed room", room); + logger.info("Group call event handler processed room", room.roomId); this.getRoomDeferred(room.roomId).resolve(); } From 83c848093fe49652aedee71d963dfe07fd6d73f2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 7 Sep 2022 15:56:38 +0100 Subject: [PATCH 250/291] MediaHandler Tests (#2646) * MediaHandler Tests, part 1 Haven't got through all the methods yet For https://github.com/vector-im/element-call/issues/544 * Didn't need these in the end * Rest of the media handler tests * getUserMediaStream takes args * use mockResolvedValue * Add .off & reuse the mock we already made * Re-use mock handler again * Move updateLocalUsermediaStream to beforeEach * add .off * Add types * Add more .offs --- spec/test-utils/webrtc.ts | 43 ++- spec/unit/webrtc/mediaHandler.spec.ts | 445 ++++++++++++++++++++++++++ src/webrtc/mediaHandler.ts | 32 +- 3 files changed, 503 insertions(+), 17 deletions(-) create mode 100644 spec/unit/webrtc/mediaHandler.spec.ts diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 4dd7e1633c9..fb8107affd8 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -160,10 +160,13 @@ export class MockRTCRtpSender { export class MockMediaStreamTrack { constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { } - stop() { } + stop = jest.fn(); listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; + public settings: MediaTrackSettings; + + getSettings(): MediaTrackSettings { return this.settings; } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own // implementation @@ -181,6 +184,8 @@ export class MockMediaStreamTrack { return t !== eventType || c !== callback; }); } + + typed(): MediaStreamTrack { return this as unknown as MediaStreamTrack; } } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own @@ -217,8 +222,12 @@ export class MockMediaStream { } removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } - clone() { - return new MockMediaStream(this.id, this.tracks); + clone(): MediaStream { + return new MockMediaStream(this.id + ".clone", this.tracks).typed(); + } + + isCloneOf(stream: MediaStream) { + return this.id === stream.id + ".clone"; } // syntactic sugar for typing @@ -231,6 +240,8 @@ export class MockMediaDeviceInfo { constructor( public kind: "audioinput" | "videoinput" | "audiooutput", ) { } + + typed(): MediaDeviceInfo { return this as unknown as MediaDeviceInfo; } } export class MockMediaHandler { @@ -267,15 +278,27 @@ export class MockMediaHandler { typed(): MediaHandler { return this as unknown as MediaHandler; } } +export class MockMediaDevices { + enumerateDevices = jest.fn, []>().mockResolvedValue([ + new MockMediaDeviceInfo("audioinput").typed(), + new MockMediaDeviceInfo("videoinput").typed(), + ]); + + getUserMedia = jest.fn, [MediaStreamConstraints]>().mockReturnValue( + Promise.resolve(new MockMediaStream("local_stream").typed()), + ); + + getDisplayMedia = jest.fn, [DisplayMediaStreamConstraints]>().mockReturnValue( + Promise.resolve(new MockMediaStream("local_display_stream").typed()), + ); + + typed(): MediaDevices { return this as unknown as MediaDevices; } +} + export function installWebRTCMocks() { global.navigator = { - mediaDevices: { - // @ts-ignore Mock - getUserMedia: () => new MockMediaStream("local_stream"), - // @ts-ignore Mock - enumerateDevices: async () => [new MockMediaDeviceInfo("audio"), new MockMediaDeviceInfo("video")], - }, - }; + mediaDevices: new MockMediaDevices().typed(), + } as unknown as Navigator; global.window = { // @ts-ignore Mock diff --git a/spec/unit/webrtc/mediaHandler.spec.ts b/spec/unit/webrtc/mediaHandler.spec.ts new file mode 100644 index 00000000000..94396845c96 --- /dev/null +++ b/spec/unit/webrtc/mediaHandler.spec.ts @@ -0,0 +1,445 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { GroupCall, MatrixCall, MatrixClient } from "../../../src"; +import { MediaHandler, MediaHandlerEvent } from "../../../src/webrtc/mediaHandler"; +import { MockMediaDeviceInfo, MockMediaDevices, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; + +const FAKE_AUDIO_INPUT_ID = "aaaaaaaa"; +const FAKE_VIDEO_INPUT_ID = "vvvvvvvv"; +const FAKE_DESKTOP_SOURCE_ID = "ddddddd"; + +describe('Media Handler', function() { + let mockMediaDevices: MockMediaDevices; + let mediaHandler: MediaHandler; + let calls: Map; + let groupCalls: Map; + + beforeEach(() => { + mockMediaDevices = new MockMediaDevices(); + + global.navigator = { + mediaDevices: mockMediaDevices.typed(), + } as unknown as Navigator; + + calls = new Map(); + groupCalls = new Map(); + + mediaHandler = new MediaHandler({ + callEventHandler: { + calls, + }, + groupCallEventHandler: { + groupCalls, + }, + } as unknown as MatrixClient); + }); + + it("does not trigger update after restore media settings ", () => { + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + expect(mockMediaDevices.getUserMedia).not.toHaveBeenCalled(); + }); + + it("sets device IDs on restore media settings", async () => { + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + it("sets audio device ID", async () => { + await mediaHandler.setAudioInput(FAKE_AUDIO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, false); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + })); + }); + + it("sets video device ID", async () => { + await mediaHandler.setVideoInput(FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(false, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + it("sets media inputs", async () => { + await mediaHandler.setMediaInputs(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + describe("updateLocalUsermediaStreams", () => { + let localStreamsChangedHandler: jest.Mock; + + beforeEach(() => { + localStreamsChangedHandler = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, localStreamsChangedHandler); + }); + + afterEach(() => { + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, localStreamsChangedHandler); + }); + + it("does nothing if it has no streams", async () => { + mediaHandler.updateLocalUsermediaStreams(); + expect(mockMediaDevices.getUserMedia).not.toHaveBeenCalled(); + }); + + it("does not emit LocalStreamsChanged if it had no streams", async () => { + await mediaHandler.updateLocalUsermediaStreams(); + + expect(localStreamsChangedHandler).not.toHaveBeenCalled(); + }); + + describe("with existing streams", () => { + let stopTrack: jest.Mock; + let updateLocalUsermediaStream: jest.Mock; + + beforeEach(() => { + stopTrack = jest.fn(); + + mediaHandler.userMediaStreams = [ + { + getTracks: () => [{ + stop: stopTrack, + } as unknown as MediaStreamTrack], + } as unknown as MediaStream, + ]; + + updateLocalUsermediaStream = jest.fn(); + }); + + it("stops existing streams", async () => { + mediaHandler.updateLocalUsermediaStreams(); + expect(stopTrack).toHaveBeenCalled(); + }); + + it("replaces streams on calls", async () => { + calls.set("some_call", { + hasLocalUserMediaAudioTrack: true, + hasLocalUserMediaVideoTrack: true, + callHasEnded: jest.fn().mockReturnValue(false), + updateLocalUsermediaStream, + } as unknown as MatrixCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).toHaveBeenCalled(); + }); + + it("doesn't replace streams on ended calls", async () => { + calls.set("some_call", { + hasLocalUserMediaAudioTrack: true, + hasLocalUserMediaVideoTrack: true, + callHasEnded: jest.fn().mockReturnValue(true), + updateLocalUsermediaStream, + } as unknown as MatrixCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).not.toHaveBeenCalled(); + }); + + it("replaces streams on group calls", async () => { + groupCalls.set("some_group_call", { + localCallFeed: {}, + updateLocalUsermediaStream, + } as unknown as GroupCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).toHaveBeenCalled(); + }); + + it("doesn't replace streams on group calls with no localCallFeed", async () => { + groupCalls.set("some_group_call", { + localCallFeed: null, + updateLocalUsermediaStream, + } as unknown as GroupCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).not.toHaveBeenCalled(); + }); + + it("emits LocalStreamsChanged", async () => { + await mediaHandler.updateLocalUsermediaStreams(); + + expect(localStreamsChangedHandler).toHaveBeenCalled(); + }); + }); + }); + + describe("hasAudioDevice", () => { + it("returns true if the system has audio inputs", async () => { + expect(await mediaHandler.hasAudioDevice()).toEqual(true); + }); + + it("returns false if the system has no audio inputs", async () => { + mockMediaDevices.enumerateDevices.mockReturnValue(Promise.resolve([ + new MockMediaDeviceInfo("videoinput").typed(), + ])); + expect(await mediaHandler.hasAudioDevice()).toEqual(false); + }); + }); + + describe("hasVideoDevice", () => { + it("returns true if the system has video inputs", async () => { + expect(await mediaHandler.hasVideoDevice()).toEqual(true); + }); + + it("returns false if the system has no video inputs", async () => { + mockMediaDevices.enumerateDevices.mockReturnValue(Promise.resolve([ + new MockMediaDeviceInfo("audioinput").typed(), + ])); + expect(await mediaHandler.hasVideoDevice()).toEqual(false); + }); + }); + + describe("getUserMediaStream", () => { + beforeEach(() => { + // replace this with one that returns a new object each time so we can + // tell whether we've ended up with the same stream + mockMediaDevices.getUserMedia.mockImplementation((constraints: MediaStreamConstraints) => { + const stream = new MockMediaStream("local_stream"); + if (constraints.audio) { + const track = new MockMediaStreamTrack("audio_track", "audio"); + track.settings = { deviceId: FAKE_AUDIO_INPUT_ID }; + stream.addTrack(track); + } + if (constraints.video) { + const track = new MockMediaStreamTrack("video_track", "video"); + track.settings = { deviceId: FAKE_VIDEO_INPUT_ID }; + stream.addTrack(track); + } + + return Promise.resolve(stream.typed()); + }); + + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + }); + + it("returns the same stream for reusable streams", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + }); + + it("doesn't re-use stream if reusable is false", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false, false); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("doesn't re-use stream if existing stream lacks audio", async () => { + const stream1 = await mediaHandler.getUserMediaStream(false, true); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("doesn't re-use stream if existing stream lacks video", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false); + const stream2 = await mediaHandler.getUserMediaStream(false, true) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("strips unwanted audio tracks from re-used stream", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, true); + const stream2 = await mediaHandler.getUserMediaStream(false, true) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + expect(stream2.getAudioTracks().length).toEqual(0); + }); + + it("strips unwanted video tracks from re-used stream", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, true); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + expect(stream2.getVideoTracks().length).toEqual(0); + }); + }); + + describe("getScreensharingStream", () => { + it("gets any screen sharing stream when called with no args", async () => { + const stream = await mediaHandler.getScreensharingStream(); + expect(stream).toBeTruthy(); + expect(stream.getTracks()).toBeTruthy(); + }); + + it("re-uses streams", async () => { + const stream = await mediaHandler.getScreensharingStream(undefined, true); + + expect(mockMediaDevices.getDisplayMedia).toHaveBeenCalled(); + mockMediaDevices.getDisplayMedia.mockClear(); + + const stream2 = await mediaHandler.getScreensharingStream() as unknown as MockMediaStream; + + expect(mockMediaDevices.getDisplayMedia).not.toHaveBeenCalled(); + + expect(stream2.isCloneOf(stream)).toEqual(true); + }); + + it("passes through desktopCapturerSourceId for Electron", async () => { + await mediaHandler.getScreensharingStream({ + desktopCapturerSourceId: FAKE_DESKTOP_SOURCE_ID, + }); + + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + video: { + mandatory: expect.objectContaining({ + chromeMediaSource: "desktop", + chromeMediaSourceId: FAKE_DESKTOP_SOURCE_ID, + }), + }, + })); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + await mediaHandler.getScreensharingStream(); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopUserMediaStream", () => { + let stream: MediaStream; + + beforeEach(async () => { + stream = await mediaHandler.getUserMediaStream(true, false); + }); + + it("stops tracks on streams", async () => { + const mockTrack = new MockMediaStreamTrack("audio_track", "audio"); + stream.addTrack(mockTrack.typed()); + + mediaHandler.stopUserMediaStream(stream); + expect(mockTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.userMediaStreams).toContain(stream); + mediaHandler.stopUserMediaStream(stream); + expect(mediaHandler.userMediaStreams).not.toContain(stream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopUserMediaStream(stream); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopUserMediaStream", () => { + let stream: MediaStream; + + beforeEach(async () => { + stream = await mediaHandler.getScreensharingStream(); + }); + + it("stops tracks on streams", async () => { + const mockTrack = new MockMediaStreamTrack("audio_track", "audio"); + stream.addTrack(mockTrack.typed()); + + mediaHandler.stopScreensharingStream(stream); + expect(mockTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.screensharingStreams).toContain(stream); + mediaHandler.stopScreensharingStream(stream); + expect(mediaHandler.screensharingStreams).not.toContain(stream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopScreensharingStream(stream); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopAllStreams", () => { + let userMediaStream: MediaStream; + let screenSharingStream: MediaStream; + + beforeEach(async () => { + userMediaStream = await mediaHandler.getUserMediaStream(true, false); + screenSharingStream = await mediaHandler.getScreensharingStream(); + }); + + it("stops tracks on streams", async () => { + const mockUserMediaTrack = new MockMediaStreamTrack("audio_track", "audio"); + userMediaStream.addTrack(mockUserMediaTrack.typed()); + + const mockScreenshareTrack = new MockMediaStreamTrack("audio_track", "audio"); + screenSharingStream.addTrack(mockScreenshareTrack.typed()); + + mediaHandler.stopAllStreams(); + + expect(mockUserMediaTrack.stop).toHaveBeenCalled(); + expect(mockScreenshareTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.userMediaStreams).toContain(userMediaStream); + expect(mediaHandler.screensharingStreams).toContain(screenSharingStream); + mediaHandler.stopAllStreams(); + expect(mediaHandler.userMediaStreams).not.toContain(userMediaStream); + expect(mediaHandler.screensharingStreams).not.toContain(screenSharingStream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopAllStreams(); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); +}); diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 1b55cfbb908..2eba5f2f534 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -179,13 +179,31 @@ export class MediaHandler extends TypedEventEmitter< let stream: MediaStream; - if ( - !this.localUserMediaStream || - (this.localUserMediaStream.getAudioTracks().length === 0 && shouldRequestAudio) || - (this.localUserMediaStream.getVideoTracks().length === 0 && shouldRequestVideo) || - (this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput) || - (this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput) - ) { + let canReuseStream = true; + if (this.localUserMediaStream) { + // This code checks that the device ID is the same as the localUserMediaStream stream, but we update + // the localUserMediaStream whenever the device ID changes (apart from when restoring) so it's not + // clear why this would ever be different, unless there's a race. + if (shouldRequestAudio) { + if ( + this.localUserMediaStream.getAudioTracks().length === 0 || + this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput + ) { + canReuseStream = false; + } + } + if (shouldRequestVideo) { + if ( + this.localUserMediaStream.getVideoTracks().length === 0 || + this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput) { + canReuseStream = false; + } + } + } else { + canReuseStream = false; + } + + if (!canReuseStream) { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); stream = await navigator.mediaDevices.getUserMedia(constraints); logger.log(`mediaHandler getUserMediaStream streamId ${stream.id} shouldRequestAudio ${ From aebe26db969f0138bed2a46d3382f7810d449a15 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 8 Sep 2022 21:46:28 +0100 Subject: [PATCH 251/291] GroupCallEventhandler Tests (#2654) * GroupCallEventhandler Tests Fixes https://github.com/vector-im/element-call/issues/545 * Fix long line * Fix strict mode error Co-authored-by: Robin * Fix typo Co-authored-by: Robin Co-authored-by: Robin --- spec/test-utils/webrtc.ts | 95 ++++++++++ spec/unit/webrtc/groupCall.spec.ts | 53 +----- .../unit/webrtc/groupCallEventHandler.spec.ts | 162 ++++++++++++++++++ src/webrtc/groupCallEventHandler.ts | 3 +- 4 files changed, 261 insertions(+), 52 deletions(-) create mode 100644 spec/unit/webrtc/groupCallEventHandler.spec.ts diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index fb8107affd8..0d3ec6fb754 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -14,6 +14,27 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { + ClientEvent, + ClientEventHandlerMap, + EventType, + GroupCall, + GroupCallIntent, + GroupCallType, + ISendEventResponse, + MatrixClient, + MatrixEvent, + Room, + RoomStateEvent, + RoomStateEventHandlerMap, +} from "../../src"; +import { TypedEventEmitter } from "../../src/models/typed-event-emitter"; +import { ReEmitter } from "../../src/ReEmitter"; +import { SyncState } from "../../src/sync"; +import { CallEvent, CallEventHandlerMap, MatrixCall } from "../../src/webrtc/call"; +import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from "../../src/webrtc/callEventHandler"; +import { GroupCallEventHandlerMap } from "../../src/webrtc/groupCall"; +import { GroupCallEventHandlerEvent } from "../../src/webrtc/groupCallEventHandler"; import { IScreensharingOpts, MediaHandler } from "../../src/webrtc/mediaHandler"; export const DUMMY_SDP = ( @@ -295,6 +316,57 @@ export class MockMediaDevices { typed(): MediaDevices { return this as unknown as MediaDevices; } } +type EmittedEvents = CallEventHandlerEvent | CallEvent | ClientEvent | RoomStateEvent | GroupCallEventHandlerEvent; +type EmittedEventMap = CallEventHandlerEventHandlerMap & + CallEventHandlerMap & + ClientEventHandlerMap & + RoomStateEventHandlerMap & + GroupCallEventHandlerMap; + +export class MockCallMatrixClient extends TypedEventEmitter { + public mediaHandler = new MockMediaHandler(); + + constructor(public userId: string, public deviceId: string, public sessionId: string) { + super(); + } + + groupCallEventHandler = { + groupCalls: new Map(), + }; + + callEventHandler = { + calls: new Map(), + }; + + sendStateEvent = jest.fn, [ + roomId: string, eventType: EventType, content: any, statekey: string, + ]>(); + sendToDevice = jest.fn, [ + eventType: string, + contentMap: { [userId: string]: { [deviceId: string]: Record } }, + txnId?: string, + ]>(); + + getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } + + getUserId(): string { return this.userId; } + + getDeviceId(): string { return this.deviceId; } + getSessionId(): string { return this.sessionId; } + + getTurnServers = () => []; + isFallbackICEServerAllowed = () => false; + reEmitter = new ReEmitter(new TypedEventEmitter()); + getUseE2eForGroupCall = () => false; + checkTurnServers = () => null; + + getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); + + getRooms = jest.fn().mockReturnValue([]); + + typed(): MatrixClient { return this as unknown as MatrixClient; } +} + export function installWebRTCMocks() { global.navigator = { mediaDevices: new MockMediaDevices().typed(), @@ -318,3 +390,26 @@ export function installWebRTCMocks() { // @ts-ignore Mock global.RTCRtpReceiver = {}; } + +export function makeMockGroupCallStateEvent(roomId: string, groupCallId: string): MatrixEvent { + return { + getType: jest.fn().mockReturnValue(EventType.GroupCallPrefix), + getRoomId: jest.fn().mockReturnValue(roomId), + getTs: jest.fn().mockReturnValue(0), + getContent: jest.fn().mockReturnValue({ + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + }), + getStateKey: jest.fn().mockReturnValue(groupCallId), + } as unknown as MatrixEvent; +} + +export function makeMockGroupCallMemberStateEvent(roomId: string, groupCallId: string): MatrixEvent { + return { + getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix), + getRoomId: jest.fn().mockReturnValue(roomId), + getTs: jest.fn().mockReturnValue(0), + getContent: jest.fn().mockReturnValue({}), + getStateKey: jest.fn().mockReturnValue(groupCallId), + } as unknown as MatrixEvent; +} diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index dd8825e40b1..5ea4bb42090 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -18,7 +18,6 @@ import { EventType, GroupCallIntent, GroupCallType, - ISendEventResponse, MatrixCall, MatrixEvent, Room, @@ -28,19 +27,16 @@ import { GroupCall } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; import { installWebRTCMocks, - MockMediaHandler, + MockCallMatrixClient, MockMediaStream, MockMediaStreamTrack, MockRTCPeerConnection, } from '../../test-utils/webrtc'; import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; import { sleep } from "../../../src/utils"; -import { ReEmitter } from "../../../src/ReEmitter"; -import { TypedEventEmitter } from '../../../src/models/typed-event-emitter'; -import { MediaHandler } from '../../../src/webrtc/mediaHandler'; -import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from '../../../src/webrtc/callEventHandler'; +import { CallEventHandlerEvent } from '../../../src/webrtc/callEventHandler'; import { CallFeed } from '../../../src/webrtc/callFeed'; -import { CallEvent, CallEventHandlerMap, CallState } from '../../../src/webrtc/call'; +import { CallEvent, CallState } from '../../../src/webrtc/call'; import { flushPromises } from '../../test-utils/flushPromises'; const FAKE_ROOM_ID = "!fake:test.dummy"; @@ -107,49 +103,6 @@ const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise { - public mediaHandler = new MockMediaHandler(); - - constructor(public userId: string, public deviceId: string, public sessionId: string) { - super(); - } - - groupCallEventHandler = { - groupCalls: new Map(), - }; - - callEventHandler = { - calls: new Map(), - }; - - sendStateEvent = jest.fn, [ - roomId: string, eventType: EventType, content: any, statekey: string, - ]>(); - sendToDevice = jest.fn, [ - eventType: string, - contentMap: { [userId: string]: { [deviceId: string]: Record } }, - txnId?: string, - ]>(); - - getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } - - getUserId(): string { return this.userId; } - - getDeviceId(): string { return this.deviceId; } - getSessionId(): string { return this.sessionId; } - - getTurnServers = () => []; - isFallbackICEServerAllowed = () => false; - reEmitter = new ReEmitter(new TypedEventEmitter()); - getUseE2eForGroupCall = () => false; - checkTurnServers = () => null; - - typed(): MatrixClient { return this as unknown as MatrixClient; } -} - class MockCall { constructor(public roomId: string, public groupCallId: string) { } diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts new file mode 100644 index 00000000000..cc9217f8dd1 --- /dev/null +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -0,0 +1,162 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ClientEvent, GroupCall, Room, RoomState, RoomStateEvent } from "../../../src"; +import { SyncState } from "../../../src/sync"; +import { GroupCallEventHandler, GroupCallEventHandlerEvent } from "../../../src/webrtc/groupCallEventHandler"; +import { flushPromises } from "../../test-utils/flushPromises"; +import { + makeMockGroupCallMemberStateEvent, + makeMockGroupCallStateEvent, + MockCallMatrixClient, +} from "../../test-utils/webrtc"; + +const FAKE_USER_ID = "@alice:test.dummy"; +const FAKE_DEVICE_ID = "AAAAAAA"; +const FAKE_SESSION_ID = "session1"; +const FAKE_ROOM_ID = "!roomid:test.dummy"; +const FAKE_GROUP_CALL_ID = "fakegroupcallid"; + +describe('Group Call Event Handler', function() { + let groupCallEventHandler: GroupCallEventHandler; + let mockClient: MockCallMatrixClient; + let mockRoom: Room; + + beforeEach(() => { + mockClient = new MockCallMatrixClient( + FAKE_USER_ID, FAKE_DEVICE_ID, FAKE_SESSION_ID, + ); + groupCallEventHandler = new GroupCallEventHandler(mockClient.typed()); + + mockRoom = { + roomId: FAKE_ROOM_ID, + currentState: { + getStateEvents: jest.fn().mockReturnValue([makeMockGroupCallStateEvent( + FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, + )]), + }, + } as unknown as Room; + + (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); + }); + + it("waits until client starts syncing", async () => { + mockClient.getSyncState.mockReturnValue(null); + let isStarted = false; + (async () => { + await groupCallEventHandler.start(); + isStarted = true; + })(); + + const setSyncState = async (newState: SyncState) => { + const oldState = mockClient.getSyncState(); + mockClient.getSyncState.mockReturnValue(newState); + mockClient.emit(ClientEvent.Sync, newState, oldState, undefined); + await flushPromises(); + }; + + await flushPromises(); + expect(isStarted).toEqual(false); + + await setSyncState(SyncState.Prepared); + expect(isStarted).toEqual(false); + + await setSyncState(SyncState.Syncing); + expect(isStarted).toEqual(true); + }); + + it("finds existing group calls when started", async () => { + const mockClientEmit = mockClient.emit = jest.fn(); + + mockClient.getRooms.mockReturnValue([mockRoom]); + await groupCallEventHandler.start(); + + expect(mockClientEmit).toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + + groupCallEventHandler.stop(); + }); + + it("can wait until a room is ready for group calls", async () => { + await groupCallEventHandler.start(); + + const prom = groupCallEventHandler.waitUntilRoomReadyForGroupCalls(FAKE_ROOM_ID); + let resolved = false; + + (async () => { + await prom; + resolved = true; + })(); + + expect(resolved).toEqual(false); + mockClient.emit(ClientEvent.Room, mockRoom); + + await prom; + expect(resolved).toEqual(true); + + groupCallEventHandler.stop(); + }); + + it("fires events for incoming calls", async () => { + const onIncomingGroupCall = jest.fn(); + mockClient.on(GroupCallEventHandlerEvent.Incoming, onIncomingGroupCall); + await groupCallEventHandler.start(); + + mockClient.emit( + RoomStateEvent.Events, + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, + ), + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + null, + ); + + expect(onIncomingGroupCall).toHaveBeenCalledWith(expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + })); + + mockClient.off(GroupCallEventHandlerEvent.Incoming, onIncomingGroupCall); + }); + + it("sends member events to group calls", async () => { + await groupCallEventHandler.start(); + + const mockGroupCall = { + onMemberStateChanged: jest.fn(), + }; + + groupCallEventHandler.groupCalls.set(FAKE_ROOM_ID, mockGroupCall as unknown as GroupCall); + + const mockStateEvent = makeMockGroupCallMemberStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID); + + mockClient.emit( + RoomStateEvent.Events, + mockStateEvent, + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + null, + ); + + expect(mockGroupCall.onMemberStateChanged).toHaveBeenCalledWith(mockStateEvent); + }); +}); diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index eb4cb7723ab..ddfc806e389 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -15,7 +15,6 @@ limitations under the License. */ import { MatrixEvent } from '../models/event'; -import { RoomStateEvent } from '../models/room-state'; import { MatrixClient, ClientEvent } from '../client'; import { GroupCall, @@ -24,7 +23,7 @@ import { IGroupCallDataChannelOptions, } from "./groupCall"; import { Room } from "../models/room"; -import { RoomState } from "../models/room-state"; +import { RoomState, RoomStateEvent } from "../models/room-state"; import { RoomMember } from "../models/room-member"; import { logger } from '../logger'; import { EventType } from "../@types/event"; From 36a6117ee284cefe7d16055352c9cefce30ce6b1 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 9 Sep 2022 09:53:27 -0400 Subject: [PATCH 252/291] Misc fixes for group call widgets (#2657) * Fix GroupCallEventHandler in matryoshka mode GroupCallEventHandler needs to see a 'Syncing' event before it starts handling any events, so emit one immediately in matryoshka mode. * Implement joinRoom on RoomWidgetClient Element Call has undergone some changes to how it loads rooms, meaning that this method must be implemented for the app to work in matryoshka mode. * Allow audio and video to be muted before local call feed exists This is desirable for the Element Web integration of Element Call, because we need to be able to mute our devices before ever joining the call or creating a call feed, if the users requests it. * Fix a strict mode error --- src/embedded.ts | 7 ++++++- src/webrtc/groupCall.ts | 22 ++++++++++++++-------- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/src/embedded.ts b/src/embedded.ts index 047cd0279c3..48a0b8dd448 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -131,7 +131,7 @@ export class RoomWidgetClient extends MatrixClient { }); }) ?? [], ); - this.setSyncState(SyncState.Prepared); + this.setSyncState(SyncState.Syncing); logger.info("Finished backfilling events"); // Watch for TURN servers, if requested @@ -146,6 +146,11 @@ export class RoomWidgetClient extends MatrixClient { this.lifecycle.abort(); // Signal to other async tasks that the client has stopped } + public async joinRoom(roomIdOrAlias: string): Promise { + if (roomIdOrAlias === this.roomId) return this.room; + throw new Error(`Unknown room: ${roomIdOrAlias}`); + } + public async sendStateEvent( roomId: string, eventType: string, diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 3cb24c9bed0..104eacae0c4 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -183,6 +183,8 @@ export class GroupCall extends TypedEventEmitter< private transmitTimer: ReturnType | null = null; private memberStateExpirationTimers: Map> = new Map(); private resendMemberStateTimer: ReturnType | null = null; + private initWithAudioMuted = false; + private initWithVideoMuted = false; constructor( private client: MatrixClient, @@ -261,12 +263,7 @@ export class GroupCall extends TypedEventEmitter< throw error; } - // start muted on ptt calls - if (this.isPtt) { - setTracksEnabled(stream.getAudioTracks(), false); - } - - const userId = this.client.getUserId(); + const userId = this.client.getUserId()!; const callFeed = new CallFeed({ client: this.client, @@ -274,10 +271,13 @@ export class GroupCall extends TypedEventEmitter< userId, stream, purpose: SDPStreamMetadataPurpose.Usermedia, - audioMuted: stream.getAudioTracks().length === 0 || this.isPtt, - videoMuted: stream.getVideoTracks().length === 0, + audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, + videoMuted: this.initWithVideoMuted || stream.getVideoTracks().length === 0, }); + setTracksEnabled(stream.getAudioTracks(), !callFeed.isAudioMuted()); + setTracksEnabled(stream.getVideoTracks(), !callFeed.isVideoMuted()); + this.localCallFeed = callFeed; this.addUserMediaFeed(callFeed); @@ -496,6 +496,9 @@ export class GroupCall extends TypedEventEmitter< // given to any of the actual calls, so these tracks don't actually go // anywhere. Let's do it anyway to avoid confusion. setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); + } else { + logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted no stream muted ${muted}`); + this.initWithAudioMuted = muted; } for (const call of this.calls) { @@ -532,6 +535,9 @@ export class GroupCall extends TypedEventEmitter< this.localCallFeed.stream.id} muted ${muted}`); this.localCallFeed.setAudioVideoMuted(null, muted); setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); + } else { + logger.log(`groupCall ${this.groupCallId} setLocalVideoMuted no stream muted ${muted}`); + this.initWithVideoMuted = muted; } for (const call of this.calls) { From 02f6a09bcf7bc2214a2e50b22e446ed2e7c664be Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Sep 2022 18:57:25 +0100 Subject: [PATCH 253/291] Test active speaker events (#2658) Fixes https://github.com/vector-im/element-call/issues/527 --- spec/unit/webrtc/groupCall.spec.ts | 71 +++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 5ea4bb42090..74563bda0ec 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -23,7 +23,7 @@ import { Room, RoomMember, } from '../../../src'; -import { GroupCall } from "../../../src/webrtc/groupCall"; +import { GroupCall, GroupCallEvent } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; import { installWebRTCMocks, @@ -48,6 +48,7 @@ const FAKE_SESSION_ID_1 = "alice1"; const FAKE_USER_ID_2 = "@bob:test.dummy"; const FAKE_DEVICE_ID_2 = "@BBBBBB"; const FAKE_SESSION_ID_2 = "bob1"; +const FAKE_USER_ID_3 = "@charlie:test.dummy"; const FAKE_STATE_EVENTS = [ { getContent: () => ({ @@ -129,6 +130,8 @@ class MockCall { userId: this.opponentUserId, }; } + + typed(): MatrixCall { return this as unknown as MatrixCall; } } describe('Group Call', function() { @@ -885,4 +888,70 @@ describe('Group Call', function() { groupCall.terminate(); }); }); + + describe("active speaker events", () => { + let room: Room; + let groupCall: GroupCall; + let mediaFeed1: CallFeed; + let mediaFeed2: CallFeed; + let onActiveSpeakerEvent: jest.Mock; + + beforeEach(async () => { + jest.useFakeTimers(); + + const mockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + + room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1); + groupCall = await createAndEnterGroupCall(mockClient.typed(), room); + + mediaFeed1 = new CallFeed({ + client: mockClient.typed(), + roomId: FAKE_ROOM_ID, + userId: FAKE_USER_ID_2, + stream: (new MockMediaStream("foo", [])).typed(), + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: true, + }); + groupCall.userMediaFeeds.push(mediaFeed1); + + mediaFeed2 = new CallFeed({ + client: mockClient.typed(), + roomId: FAKE_ROOM_ID, + userId: FAKE_USER_ID_3, + stream: (new MockMediaStream("foo", [])).typed(), + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: true, + }); + groupCall.userMediaFeeds.push(mediaFeed2); + + onActiveSpeakerEvent = jest.fn(); + groupCall.on(GroupCallEvent.ActiveSpeakerChanged, onActiveSpeakerEvent); + }); + + afterEach(() => { + groupCall.off(GroupCallEvent.ActiveSpeakerChanged, onActiveSpeakerEvent); + + jest.useRealTimers(); + }); + + it("fires active speaker events when a user is speaking", async () => { + mediaFeed1.speakingVolumeSamples = [100, 100]; + mediaFeed2.speakingVolumeSamples = [0, 0]; + + jest.runOnlyPendingTimers(); + expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_2); + expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_2); + + mediaFeed1.speakingVolumeSamples = [0, 0]; + mediaFeed2.speakingVolumeSamples = [100, 100]; + + jest.runOnlyPendingTimers(); + expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_3); + expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_3); + }); + }); }); From 00629e6dc9fecc4154c0f7f8dbea0d2cb1c910c2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 9 Sep 2022 21:15:34 +0100 Subject: [PATCH 254/291] Test fallback screensharing (#2659) * Test fallback screensharing * Test replacetrack is called * Unused import * Return type * Fix other test after new track IDs --- spec/test-utils/webrtc.ts | 23 +++++++++---- spec/unit/webrtc/call.spec.ts | 62 ++++++++++++++++++++++++++++++----- 2 files changed, 70 insertions(+), 15 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 0d3ec6fb754..fec018b7c0d 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -104,13 +104,18 @@ export class MockRTCPeerConnection { private onReadyToNegotiate: () => void; localDescription: RTCSessionDescription; signalingState: RTCSignalingState = "stable"; + public senders: MockRTCRtpSender[] = []; - public static triggerAllNegotiations() { + public static triggerAllNegotiations(): void { for (const inst of this.instances) { inst.doNegotiation(); } } + public static hasAnyPendingNegotiations(): boolean { + return this.instances.some(i => i.needsNegotiation); + } + public static resetInstances() { this.instances = []; } @@ -153,10 +158,12 @@ export class MockRTCPeerConnection { } close() { } getStats() { return []; } - addTrack(track: MockMediaStreamTrack) { + addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { this.needsNegotiation = true; this.onReadyToNegotiate(); - return new MockRTCRtpSender(track); + const newSender = new MockRTCRtpSender(track); + this.senders.push(newSender); + return newSender; } removeTrack() { @@ -164,6 +171,8 @@ export class MockRTCPeerConnection { this.onReadyToNegotiate(); } + getSenders(): MockRTCRtpSender[] { return this.senders; } + doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { this.needsNegotiation = false; @@ -271,8 +280,8 @@ export class MockMediaHandler { getUserMediaStream(audio: boolean, video: boolean) { const tracks = []; - if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); - if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); + if (audio) tracks.push(new MockMediaStreamTrack("usermedia_audio_track", "audio")); + if (video) tracks.push(new MockMediaStreamTrack("usermedia_video_track", "video")); const stream = new MockMediaStream(USERMEDIA_STREAM_ID, tracks); this.userMediaStreams.push(stream); @@ -282,8 +291,8 @@ export class MockMediaHandler { stream.isStopped = true; } getScreensharingStream = jest.fn((opts?: IScreensharingOpts) => { - const tracks = [new MockMediaStreamTrack("video_track", "video")]; - if (opts?.audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); + const tracks = [new MockMediaStreamTrack("screenshare_video_track", "video")]; + if (opts?.audio) tracks.push(new MockMediaStreamTrack("screenshare_audio_track", "audio")); const stream = new MockMediaStream(SCREENSHARE_STREAM_ID, tracks); this.screensharingStreams.push(stream); diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 55926e28e53..fc1a5120d8c 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -389,14 +389,14 @@ describe('Call', function() { const usermediaSenders: Array = (call as any).usermediaSenders; - expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("audio_track"); - expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); + expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("usermedia_audio_track"); + expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("usermedia_video_track"); expect(usermediaSenders.find((sender) => { return sender?.track?.kind === "audio"; - }).track.id).toBe("audio_track"); + }).track.id).toBe("usermedia_audio_track"); expect(usermediaSenders.find((sender) => { return sender?.track?.kind === "video"; - }).track.id).toBe("video_track"); + }).track.id).toBe("usermedia_video_track"); }); it("should handle SDPStreamMetadata changes", async () => { @@ -925,8 +925,8 @@ describe('Call', function() { await call.setScreensharingEnabled(true); expect( - call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length, - ).toEqual(1); + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), + ).toHaveLength(1); mockSendEvent.mockReset(); const sendNegotiatePromise = new Promise(resolve => { @@ -955,8 +955,54 @@ describe('Call', function() { await call.setScreensharingEnabled(false); expect( - call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length, - ).toEqual(0); + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), + ).toHaveLength(0); }); }); + + it("falls back to replaceTrack for opponents that don't support stream metadata", async () => { + await startVideoCall(client, call); + + await call.onAnswerReceived(makeMockEvent("@test:foo", { + "version": 1, + "call_id": call.callId, + "party_id": 'party_id', + "answer": { + sdp: DUMMY_SDP, + }, + })); + + MockRTCPeerConnection.triggerAllNegotiations(); + + const mockVideoSender = call.peerConn.getSenders().find(s => s.track.kind === "video"); + const mockReplaceTrack = mockVideoSender.replaceTrack = jest.fn(); + + await call.setScreensharingEnabled(true); + + // our local feed should still reflect the purpose of the feed (ie. screenshare) + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length, + ).toEqual(1); + + // but we should not have re-negotiated + expect(MockRTCPeerConnection.hasAnyPendingNegotiations()).toEqual(false); + + expect(mockReplaceTrack).toHaveBeenCalledWith(expect.objectContaining({ + id: "screenshare_video_track", + })); + mockReplaceTrack.mockClear(); + + await call.setScreensharingEnabled(false); + + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), + ).toHaveLength(0); + expect(call.getLocalFeeds()).toHaveLength(1); + + expect(MockRTCPeerConnection.hasAnyPendingNegotiations()).toEqual(false); + + expect(mockReplaceTrack).toHaveBeenCalledWith(expect.objectContaining({ + id: "usermedia_video_track", + })); + }); }); From 37118991f5a491ba52414fa407ee392b5e850542 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 12 Sep 2022 09:40:28 +0100 Subject: [PATCH 255/291] Add test for removing RTX codec (#2660) * Add test for removing RTX codec * Use mocked to cast --- spec/test-utils/webrtc.ts | 18 ++++++++++++++++- spec/unit/webrtc/call.spec.ts | 37 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index fec018b7c0d..a0bdba354ca 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -173,6 +173,8 @@ export class MockRTCPeerConnection { getSenders(): MockRTCRtpSender[] { return this.senders; } + getTransceivers = jest.fn().mockReturnValue([]); + doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { this.needsNegotiation = false; @@ -185,6 +187,7 @@ export class MockRTCRtpSender { constructor(public track: MockMediaStreamTrack) { } replaceTrack(track: MockMediaStreamTrack) { this.track = track; } + setCodecPreferences(prefs: RTCRtpCodecCapability[]): void {} } export class MockMediaStreamTrack { @@ -397,7 +400,20 @@ export function installWebRTCMocks() { global.AudioContext = MockAudioContext; // @ts-ignore Mock - global.RTCRtpReceiver = {}; + global.RTCRtpReceiver = { + getCapabilities: jest.fn().mockReturnValue({ + codecs: [], + headerExtensions: [], + }), + }; + + // @ts-ignore Mock + global.RTCRtpSender = { + getCapabilities: jest.fn().mockReturnValue({ + codecs: [], + headerExtensions: [], + }), + }; } export function makeMockGroupCallStateEvent(roomId: string, groupCallId: string): MatrixEvent { diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index fc1a5120d8c..156efbfed40 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -14,6 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { mocked } from "jest-mock"; + import { TestClient } from '../../TestClient'; import { MatrixCall, @@ -32,6 +34,7 @@ import { installWebRTCMocks, MockRTCPeerConnection, SCREENSHARE_STREAM_ID, + MockRTCRtpSender, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { Callback, EventType, IContent, MatrixEvent, Room } from "../../../src"; @@ -958,6 +961,40 @@ describe('Call', function() { call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), ).toHaveLength(0); }); + + it("removes RTX codec from screen sharing transcievers", async () => { + mocked(global.RTCRtpSender.getCapabilities).mockReturnValue({ + codecs: [ + { mimeType: "video/rtx", clockRate: 90000 }, + { mimeType: "video/somethingelse", clockRate: 90000 }, + ], + headerExtensions: [], + }); + + const prom = new Promise(resolve => { + const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + mockPeerConn.addTrack = jest.fn().mockImplementation((track: MockMediaStreamTrack) => { + const mockSender = new MockRTCRtpSender(track); + mockPeerConn.getTransceivers.mockReturnValue([{ + sender: mockSender, + setCodecPreferences: (prefs: RTCRtpCodecCapability[]) => { + expect(prefs).toEqual([ + expect.objectContaining({ mimeType: "video/somethingelse" }), + ]); + + resolve(); + }, + }]); + + return mockSender; + }); + }); + + await call.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + + await prom; + }); }); it("falls back to replaceTrack for opponents that don't support stream metadata", async () => { From 3e1e99f8e575f53318b7b19b3a7ce4b194d229a1 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 12 Sep 2022 10:34:35 +0100 Subject: [PATCH 256/291] Fix import in failed merge --- src/client.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/client.ts b/src/client.ts index 31c9044da07..ba8f94824f9 100644 --- a/src/client.ts +++ b/src/client.ts @@ -192,7 +192,7 @@ import { } from "./webrtc/groupCall"; import { MediaHandler } from "./webrtc/mediaHandler"; import { GroupCallEventHandler } from "./webrtc/groupCallEventHandler"; -import { ILoginFlowsResponse, IRefreshTokenResponse, SSOAction, IRefreshTokenResponse } from "./@types/auth"; +import { ILoginFlowsResponse, IRefreshTokenResponse, SSOAction } from "./@types/auth"; import { TypedEventEmitter } from "./models/typed-event-emitter"; import { ReceiptType } from "./@types/read_receipts"; import { MSC3575SlidingSyncRequest, MSC3575SlidingSyncResponse, SlidingSync } from "./sliding-sync"; From 41cee6f1cc29c14f60bfcde709ec322aeafbebb3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Sep 2022 16:30:34 +0100 Subject: [PATCH 257/291] Fix race in creating calls (#2662) * Fix race in creating calls We ran an async function between checking for an existing call and adding the new one to the map, so it would have been possible to start creating another call while we were placing the first call. This changes the code to add the call to the map as soon as we've created it. Also adds more logging. * Switch to logger.debug * Fix unit tests --- spec/unit/webrtc/groupCall.spec.ts | 2 ++ src/webrtc/groupCall.ts | 19 ++++++++++++------- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 74563bda0ec..724778ce624 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -533,6 +533,7 @@ describe('Group Call', function() { let newCall: MatrixCall; while ( (newCall = groupCall1.getCallByUserId(client2.userId)) === undefined || + newCall.peerConn === undefined || newCall.callId == oldCall.callId ) { await flushPromises(); @@ -643,6 +644,7 @@ describe('Group Call', function() { groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); const setAVMutedArray = groupCall.calls.map(call => { call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); + call.localUsermediaFeed.isVideoMuted = jest.fn().mockReturnValue(true); return call.localUsermediaFeed.setAudioVideoMuted; }); const tracksArray = groupCall.calls.reduce((acc, call) => { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 104eacae0c4..b96ebed7c51 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -856,12 +856,22 @@ export class GroupCall extends TypedEventEmitter< }, ); + if (existingCall) { + logger.debug(`Replacing call ${existingCall.callId} to ${member.userId} with ${newCall.callId}`); + this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); + } else { + logger.debug(`Adding call ${newCall.callId} to ${member.userId}`); + this.addCall(newCall); + } + newCall.isPtt = this.isPtt; const requestScreenshareFeed = opponentDevice.feeds.some( (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); - logger.log(`Placing call to ${member.userId}.`); + logger.debug( + `Placing call to ${member.userId}/${opponentDevice.device_id} session ID ${opponentDevice.session_id}.`, + ); try { await newCall.placeCallWithCallFeeds( @@ -881,18 +891,13 @@ export class GroupCall extends TypedEventEmitter< ), ); } + this.removeCall(newCall, CallErrorCode.SignallingFailed); return; } if (this.dataChannelsEnabled) { newCall.createDataChannel("datachannel", this.dataChannelOptions); } - - if (existingCall) { - this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); - } else { - this.addCall(newCall); - } }; public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { From c605310b871048a92e43221e7a1efb5b36d0a421 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 13 Sep 2022 20:02:14 +0100 Subject: [PATCH 258/291] Prevent exception when muting (#2667) Fixes https://github.com/vector-im/element-call/issues/578 --- src/webrtc/groupCall.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index b96ebed7c51..bf8022c2b9d 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -476,7 +476,7 @@ export class GroupCall extends TypedEventEmitter< } for (const call of this.calls) { - call.localUsermediaFeed.setAudioVideoMuted(muted, null); + call.localUsermediaFeed?.setAudioVideoMuted(muted, null); } if (sendUpdatesBefore) { From 586a313c8d2fd5e8982459b6e31d27c09d5066b8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 14 Sep 2022 09:42:57 +0100 Subject: [PATCH 259/291] Add tests for call answering / candidate sending (#2666) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add tests for call answering / candidate sending * Remopve unused stuff * Capitalise Co-authored-by: Šimon Brandner * Capitalisation * Capitalise * Fix typescript strict error * Actually fix TS strict error(?) * TS strict mode try 3 Co-authored-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 7 +++- spec/unit/webrtc/call.spec.ts | 71 +++++++++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index a0bdba354ca..5d994a4545b 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -99,6 +99,7 @@ export class MockRTCPeerConnection { private static instances: MockRTCPeerConnection[] = []; private negotiationNeededListener: () => void; + public iceCandidateListener?: (e: RTCPeerConnectionIceEvent) => void; private needsNegotiation = false; public readyToNegotiate: Promise; private onReadyToNegotiate: () => void; @@ -135,7 +136,11 @@ export class MockRTCPeerConnection { } addEventListener(type: string, listener: () => void) { - if (type === 'negotiationneeded') this.negotiationNeededListener = listener; + if (type === 'negotiationneeded') { + this.negotiationNeededListener = listener; + } else if (type == 'icecandidate') { + this.iceCandidateListener = listener; + } } createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } createOffer() { diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 156efbfed40..da80bc2c19b 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -887,6 +887,77 @@ describe('Call', function() { }); }); + describe("answering calls", () => { + beforeEach(async () => { + await fakeIncomingCall(client, call, "1"); + }); + + const untilEventSent = async (...args) => { + const maxTries = 20; + + for (let tries = 0; tries < maxTries; ++tries) { + if (tries) { + await new Promise(resolve => { + setTimeout(resolve, 100); + }); + } + try { + expect(client.client.sendEvent).toHaveBeenCalledWith(...args); + return; + } catch (e) { + if (tries == maxTries - 1) { + throw e; + } + } + } + }; + + it("sends an answer event", async () => { + await call.answer(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallAnswer, + expect.objectContaining({ + call_id: call.callId, + answer: expect.objectContaining({ + type: 'offer', + }), + }), + ); + }); + + it("sends ICE candidates as separate events if they arrive after the answer", async () => { + const fakeCandidateString = "here is a fake candidate!"; + + await call.answer(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallAnswer, + expect.objectContaining({}), + ); + + const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + mockPeerConn.iceCandidateListener!({ + candidate: { + candidate: fakeCandidateString, + sdpMLineIndex: 0, + sdpMid: '0', + toJSON: jest.fn().mockReturnValue(fakeCandidateString), + }, + } as unknown as RTCPeerConnectionIceEvent); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + }); + }); + it("times out an incoming call", async () => { jest.useFakeTimers(); await fakeIncomingCall(client, call, "1"); From f52c5eb6676c2343c6b87f29b1de27b181aaf204 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 14 Sep 2022 09:53:07 +0100 Subject: [PATCH 260/291] Unused imports from merge --- src/client.ts | 2 +- src/utils.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/client.ts b/src/client.ts index 581ed43976d..ba6de029182 100644 --- a/src/client.ts +++ b/src/client.ts @@ -192,7 +192,7 @@ import { } from "./webrtc/groupCall"; import { MediaHandler } from "./webrtc/mediaHandler"; import { GroupCallEventHandler } from "./webrtc/groupCallEventHandler"; -import { ILoginFlowsResponse, IRefreshTokenResponse, SSOAction } from "./@types/auth"; +import { IRefreshTokenResponse, SSOAction } from "./@types/auth"; import { TypedEventEmitter } from "./models/typed-event-emitter"; import { ReceiptType } from "./@types/read_receipts"; import { MSC3575SlidingSyncRequest, MSC3575SlidingSyncResponse, SlidingSync } from "./sliding-sync"; diff --git a/src/utils.ts b/src/utils.ts index 5d18f271d12..f51be323bbf 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -24,7 +24,7 @@ import unhomoglyph from "unhomoglyph"; import promiseRetry from "p-retry"; import type * as NodeCrypto from "crypto"; -import { MatrixEvent, MatrixClient } from "."; +import { MatrixEvent } from "."; import { M_TIMESTAMP } from "./@types/location"; import { ReceiptType } from "./@types/read_receipts"; From 6fc9827b103e50dc7f2b6daff55cc66b4a87425d Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 16 Sep 2022 09:26:37 +0100 Subject: [PATCH 261/291] Add tests for ice candidate sending (#2674) --- spec/unit/webrtc/call.spec.ts | 128 +++++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 25 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index da80bc2c19b..5504e3ae307 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -37,7 +37,7 @@ import { MockRTCRtpSender, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; -import { Callback, EventType, IContent, MatrixEvent, Room } from "../../../src"; +import { Callback, EventType, IContent, ISendEventResponse, MatrixEvent, Room } from "../../../src"; const FAKE_ROOM_ID = "!foo:bar"; const CALL_LIFETIME = 60000; @@ -99,7 +99,7 @@ describe('Call', function() { let prevDocument: Document; let prevWindow: Window & typeof globalThis; // We retain a reference to this in the correct Mock type - let mockSendEvent: jest.Mock]>; + let mockSendEvent: jest.Mock, [string, string, IContent, string, Callback]>; beforeEach(function() { prevNavigator = global.navigator; @@ -888,6 +888,8 @@ describe('Call', function() { }); describe("answering calls", () => { + const realSetTimeout = setTimeout; + beforeEach(async () => { await fakeIncomingCall(client, call, "1"); }); @@ -898,11 +900,14 @@ describe('Call', function() { for (let tries = 0; tries < maxTries; ++tries) { if (tries) { await new Promise(resolve => { - setTimeout(resolve, 100); + realSetTimeout(resolve, 100); }); } + // We might not always be in fake timer mode, but it's + // fine to run this if not, so we just call it anyway. + jest.runOnlyPendingTimers(); try { - expect(client.client.sendEvent).toHaveBeenCalledWith(...args); + expect(mockSendEvent).toHaveBeenCalledWith(...args); return; } catch (e) { if (tries == maxTries - 1) { @@ -926,35 +931,107 @@ describe('Call', function() { ); }); - it("sends ICE candidates as separate events if they arrive after the answer", async () => { + describe("ICE candidate sending", () => { + let mockPeerConn; const fakeCandidateString = "here is a fake candidate!"; - - await call.answer(); - await untilEventSent( - FAKE_ROOM_ID, - EventType.CallAnswer, - expect.objectContaining({}), - ); - - const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; - mockPeerConn.iceCandidateListener!({ + const fakeCandidateEvent = { candidate: { candidate: fakeCandidateString, sdpMLineIndex: 0, sdpMid: '0', toJSON: jest.fn().mockReturnValue(fakeCandidateString), }, - } as unknown as RTCPeerConnectionIceEvent); + } as unknown as RTCPeerConnectionIceEvent; + + beforeEach(async () => { + await call.answer(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallAnswer, + expect.objectContaining({}), + ); + mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + }); - await untilEventSent( - FAKE_ROOM_ID, - EventType.CallCandidates, - expect.objectContaining({ - candidates: [ - fakeCandidateString, - ], - }), - ); + afterEach(() => { + jest.useRealTimers(); + }); + + it("sends ICE candidates as separate events if they arrive after the answer", async () => { + mockPeerConn!.iceCandidateListener!(fakeCandidateEvent); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + }); + + it("retries sending ICE candidates", async () => { + jest.useFakeTimers(); + + mockSendEvent.mockRejectedValueOnce(new Error("Fake error")); + + mockPeerConn!.iceCandidateListener!(fakeCandidateEvent); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + + mockSendEvent.mockClear(); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + }); + + it("gives up on call after 5 attempts at sending ICE candidates", async () => { + jest.useFakeTimers(); + + mockSendEvent.mockImplementation((roomId: string, eventType: string) => { + if (eventType === EventType.CallCandidates) { + return Promise.reject(new Error()); + } else { + return Promise.resolve({ event_id: "foo" }); + } + }); + + mockPeerConn!.iceCandidateListener!(fakeCandidateEvent); + + while (!call.callHasEnded()) { + jest.runOnlyPendingTimers(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + if (!call.callHasEnded) { + mockSendEvent.mockReset(); + } + } + + expect(call.callHasEnded()).toEqual(true); + }); }); }); @@ -1006,6 +1083,7 @@ describe('Call', function() { const sendNegotiatePromise = new Promise(resolve => { mockSendEvent.mockImplementationOnce(() => { resolve(); + return Promise.resolve({ event_id: "foo" }); }); }); From de694459be1505dbf4ce96d5afa912d7dceb2987 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 16 Sep 2022 10:26:03 -0400 Subject: [PATCH 262/291] Target widget actions at a specific room (#2670) Otherwise, the RoomWidgetClient class can end up accidentally sending and receiving events from rooms it didn't intend to, if it's an always-on-screen widget. --- spec/unit/embedded.spec.ts | 23 +++++++-------- src/embedded.ts | 58 +++++++++++++++++++++++--------------- 2 files changed, 47 insertions(+), 34 deletions(-) diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index edac107b976..f704e847d13 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -42,6 +42,7 @@ class MockWidgetApi extends EventEmitter { public start = jest.fn(); public requestCapability = jest.fn(); public requestCapabilities = jest.fn(); + public requestCapabilityForRoomTimeline = jest.fn(); public requestCapabilityToSendState = jest.fn(); public requestCapabilityToReceiveState = jest.fn(); public requestCapabilityToSendToDevice = jest.fn(); @@ -86,20 +87,17 @@ describe("RoomWidgetClient", () => { it("sends", async () => { await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar"); await client.sendStateEvent("!1:example.org", "org.example.foo", { hello: "world" }, "bar"); - expect(widgetApi.sendStateEvent).toHaveBeenCalledWith("org.example.foo", "bar", { hello: "world" }); - }); - - it("refuses to send to other rooms", async () => { - await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); - expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar"); - await expect(client.sendStateEvent("!2:example.org", "org.example.foo", { hello: "world" }, "bar")) - .rejects.toBeDefined(); + expect(widgetApi.sendStateEvent).toHaveBeenCalledWith( + "org.example.foo", "bar", { hello: "world" }, "!1:example.org", + ); }); it("receives", async () => { await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); const emittedEvent = new Promise(resolve => client.once(ClientEvent.Event, resolve)); @@ -114,7 +112,8 @@ describe("RoomWidgetClient", () => { expect(await emittedSync).toEqual(SyncState.Syncing); // It should've also inserted the event into the room object const room = client.getRoom("!1:example.org"); - expect(room.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + expect(room).not.toBeNull(); + expect(room!.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); }); it("backfills", async () => { @@ -125,10 +124,12 @@ describe("RoomWidgetClient", () => { ); await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); const room = client.getRoom("!1:example.org"); - expect(room.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + expect(room).not.toBeNull(); + expect(room!.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); }); }); @@ -257,7 +258,7 @@ describe("RoomWidgetClient", () => { const emittedServer = new Promise(resolve => client.once(ClientEvent.TurnServers, resolve), ); - emitServer2(); + emitServer2!(); expect(await emittedServer).toEqual([clientServer2]); expect(client.getTurnServers()).toEqual([clientServer2]); }); diff --git a/src/embedded.ts b/src/embedded.ts index 48a0b8dd448..4a62810687f 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -24,6 +24,7 @@ import { ISendToDeviceToWidgetActionRequest, } from "matrix-widget-api"; +import type { IEvent, IContent } from "./models/event"; import { ISendEventResponse } from "./@types/requests"; import { EventType } from "./@types/event"; import { logger } from "./logger"; @@ -69,27 +70,30 @@ export class RoomWidgetClient extends MatrixClient { super(opts); // Request capabilities for the functionality this client needs to support - this.capabilities.sendState?.forEach(({ eventType, stateKey }) => - this.widgetApi.requestCapabilityToSendState(eventType, stateKey), + if (capabilities.sendState?.length || capabilities.receiveState?.length) { + widgetApi.requestCapabilityForRoomTimeline(roomId); + } + capabilities.sendState?.forEach(({ eventType, stateKey }) => + widgetApi.requestCapabilityToSendState(eventType, stateKey), ); - this.capabilities.receiveState?.forEach(({ eventType, stateKey }) => - this.widgetApi.requestCapabilityToReceiveState(eventType, stateKey), + capabilities.receiveState?.forEach(({ eventType, stateKey }) => + widgetApi.requestCapabilityToReceiveState(eventType, stateKey), ); - this.capabilities.sendToDevice?.forEach(eventType => - this.widgetApi.requestCapabilityToSendToDevice(eventType), + capabilities.sendToDevice?.forEach(eventType => + widgetApi.requestCapabilityToSendToDevice(eventType), ); - this.capabilities.receiveToDevice?.forEach(eventType => - this.widgetApi.requestCapabilityToReceiveToDevice(eventType), + capabilities.receiveToDevice?.forEach(eventType => + widgetApi.requestCapabilityToReceiveToDevice(eventType), ); - if (this.capabilities.turnServers) { - this.widgetApi.requestCapability(MatrixCapabilities.MSC3846TurnServers); + if (capabilities.turnServers) { + widgetApi.requestCapability(MatrixCapabilities.MSC3846TurnServers); } - this.widgetApi.on(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); - this.widgetApi.on(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + widgetApi.on(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); + widgetApi.on(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); // Open communication with the host - this.widgetApi.start(); + widgetApi.start(); } public async startClient(opts: IStartClientOpts = {}): Promise { @@ -121,8 +125,8 @@ export class RoomWidgetClient extends MatrixClient { // so it doesn't really matter what order we inject them in await Promise.all( this.capabilities.receiveState?.map(async ({ eventType, stateKey }) => { - const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey); - const events = rawEvents.map(rawEvent => new MatrixEvent(rawEvent)); + const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey, [this.roomId]); + const events = rawEvents.map(rawEvent => new MatrixEvent(rawEvent as Partial)); await this.syncApi.injectRoomEvents(this.room, [], events); events.forEach(event => { @@ -157,8 +161,7 @@ export class RoomWidgetClient extends MatrixClient { content: any, stateKey = "", ): Promise { - if (roomId !== this.roomId) throw new Error(`Can't send events to ${roomId}`); - return await this.widgetApi.sendStateEvent(eventType, stateKey, content); + return await this.widgetApi.sendStateEvent(eventType, stateKey, content, roomId); } public async sendToDevice( @@ -215,11 +218,20 @@ export class RoomWidgetClient extends MatrixClient { private onEvent = async (ev: CustomEvent) => { ev.preventDefault(); - const event = new MatrixEvent(ev.detail.data); - await this.syncApi.injectRoomEvents(this.room, [], [event]); - this.emit(ClientEvent.Event, event); - this.setSyncState(SyncState.Syncing); - logger.info(`Received event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + + // Verify the room ID matches, since it's possible for the client to + // send us events from other rooms if this widget is always on screen + if (ev.detail.data.room_id === this.roomId) { + const event = new MatrixEvent(ev.detail.data as Partial); + await this.syncApi.injectRoomEvents(this.room, [], [event]); + this.emit(ClientEvent.Event, event); + this.setSyncState(SyncState.Syncing); + logger.info(`Received event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + } else { + const { event_id: eventId, room_id: roomId } = ev.detail.data; + logger.info(`Received event ${eventId} for a different room ${roomId}; discarding`); + } + await this.ack(ev); }; @@ -229,7 +241,7 @@ export class RoomWidgetClient extends MatrixClient { const event = new MatrixEvent({ type: ev.detail.data.type, sender: ev.detail.data.sender, - content: ev.detail.data.content, + content: ev.detail.data.content as IContent, }); // Mark the event as encrypted if it was, using fake contents and keys since those are unknown to us if (ev.detail.data.encrypted) event.makeEncrypted(EventType.RoomMessageEncrypted, {}, "", ""); From f41b7706e489cc1b83e6b25dd50091be2b5a9083 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 16 Sep 2022 11:08:13 -0400 Subject: [PATCH 263/291] Upgrade matrix-widget-api (and fix the lockfile) (#2676) --- package.json | 2 +- yarn.lock | 63 ++++++++++++++++++++++++++++++++++++---------------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/package.json b/package.json index 3e9e362ac41..ade5563bd6e 100644 --- a/package.json +++ b/package.json @@ -101,8 +101,8 @@ "eslint-plugin-matrix-org": "^0.6.0", "exorcist": "^2.0.0", "fake-indexeddb": "^4.0.0", - "jest-environment-jsdom": "^28.1.3", "jest": "^29.0.0", + "jest-environment-jsdom": "^28.1.3", "jest-localstorage-mock": "^2.4.6", "jest-sonar-reporter": "^2.0.0", "jsdoc": "^3.6.6", diff --git a/yarn.lock b/yarn.lock index 729e2eb10dd..c28226222f9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1187,6 +1187,16 @@ slash "^3.0.0" strip-ansi "^6.0.0" +"@jest/environment@^28.1.3": + version "28.1.3" + resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-28.1.3.tgz#abed43a6b040a4c24fdcb69eab1f97589b2d663e" + integrity sha512-1bf40cMFTEkKyEf585R9Iz1WayDjHoHqvts0XFYEqyKM3cFWDpeMoqKKTAF9LSYQModPUlh8FKptoM2YcMWAXA== + dependencies: + "@jest/fake-timers" "^28.1.3" + "@jest/types" "^28.1.3" + "@types/node" "*" + jest-mock "^28.1.3" + "@jest/environment@^29.0.2": version "29.0.2" resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.0.2.tgz#9e4b6d4c9bce5bfced6f63945d8c8e571394f572" @@ -1219,6 +1229,18 @@ expect "^29.0.2" jest-snapshot "^29.0.2" +"@jest/fake-timers@^28.1.3": + version "28.1.3" + resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-28.1.3.tgz#230255b3ad0a3d4978f1d06f70685baea91c640e" + integrity sha512-D/wOkL2POHv52h+ok5Oj/1gOG9HSywdoPtFsRCUmlCILXNn5eIWmcnd3DIiWlJnpGvQtmajqBP95Ei0EimxfLw== + dependencies: + "@jest/types" "^28.1.3" + "@sinonjs/fake-timers" "^9.1.2" + "@types/node" "*" + jest-message-util "^28.1.3" + jest-mock "^28.1.3" + jest-util "^28.1.3" + "@jest/fake-timers@^29.0.2": version "29.0.2" resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.0.2.tgz#6f15f4d8eb1089d445e3f73473ddc434faa2f798" @@ -4470,6 +4492,17 @@ jest-docblock@^29.0.0: dependencies: detect-newline "^3.0.0" +jest-each@^29.0.2: + version "29.0.2" + resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.0.2.tgz#f98375a79a37761137e11d458502dfe1f00ba5b0" + integrity sha512-+sA9YjrJl35iCg0W0VCrgCVj+wGhDrrKQ+YAqJ/DHBC4gcDFAeePtRRhpJnX9gvOZ63G7gt52pwp2PesuSEx0Q== + dependencies: + "@jest/types" "^29.0.2" + chalk "^4.0.0" + jest-get-type "^29.0.0" + jest-util "^29.0.2" + pretty-format "^29.0.2" + jest-environment-jsdom@^28.1.3: version "28.1.3" resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-28.1.3.tgz#2d4e5d61b7f1d94c3bddfbb21f0308ee506c09fb" @@ -4484,22 +4517,6 @@ jest-environment-jsdom@^28.1.3: jest-util "^28.1.3" jsdom "^19.0.0" -jest-environment-node@^28.1.3: - version "28.1.3" - resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-28.1.3.tgz#7e74fe40eb645b9d56c0c4b70ca4357faa349be5" - integrity sha512-ugP6XOhEpjAEhGYvp5Xj989ns5cB1K6ZdjBYuS30umT4CQEETaxSiPcZ/E1kFktX4GkrcM4qu07IIlDYX1gp+A== - -jest-each@^29.0.2: - version "29.0.2" - resolved "https://registry.yarnpkg.com/jest-each/-/jest-each-29.0.2.tgz#f98375a79a37761137e11d458502dfe1f00ba5b0" - integrity sha512-+sA9YjrJl35iCg0W0VCrgCVj+wGhDrrKQ+YAqJ/DHBC4gcDFAeePtRRhpJnX9gvOZ63G7gt52pwp2PesuSEx0Q== - dependencies: - "@jest/types" "^29.0.2" - chalk "^4.0.0" - jest-get-type "^29.0.0" - jest-util "^29.0.2" - pretty-format "^29.0.2" - jest-environment-node@^29.0.2: version "29.0.2" resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.0.2.tgz#8196268c9f740f1d2e7ecccf212b4c1c5b0167e4" @@ -4604,6 +4621,14 @@ jest-message-util@^29.0.2: slash "^3.0.0" stack-utils "^2.0.3" +jest-mock@^28.1.3: + version "28.1.3" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-28.1.3.tgz#d4e9b1fc838bea595c77ab73672ebf513ab249da" + integrity sha512-o3J2jr6dMMWYVH4Lh/NKmDXdosrsJgi4AviS8oXLujcjpCMBb1FMsblDnOXKZKfSiHLxYub1eS0IHuRXsio9eA== + dependencies: + "@jest/types" "^28.1.3" + "@types/node" "*" + jest-mock@^29.0.2: version "29.0.2" resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.0.2.tgz#d7810966a6338aca6a440c3cd9f19276477840ad" @@ -5184,9 +5209,9 @@ matrix-mock-request@^2.1.2: expect "^28.1.0" matrix-widget-api@^1.0.0: - version "1.0.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.0.0.tgz#0cde6839cca66ad817ab12aca3490ccc8bac97d1" - integrity sha512-cy8p/8EteRPTFIAw7Q9EgPUJc2jD19ZahMR8bMKf2NkILDcjuPMC0UWnsJyB3fSnlGw+VbGepttRpULM31zX8Q== + version "1.1.1" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.1.1.tgz#d3fec45033d0cbc14387a38ba92dac4dbb1be962" + integrity sha512-gNSgmgSwvOsOcWK9k2+tOhEMYBiIMwX95vMZu0JqY7apkM02xrOzUBuPRProzN8CnbIALH7e3GAhatF6QCNvtA== dependencies: "@types/events" "^3.0.0" events "^3.2.0" From c400dd4ff86cda462a74096525005290c40d83d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 20 Sep 2022 19:40:47 +0200 Subject: [PATCH 264/291] Add a few new `GroupCall` tests (#2678) Co-authored-by: Robin --- spec/test-utils/webrtc.ts | 15 +++ spec/unit/webrtc/groupCall.spec.ts | 158 ++++++++++++++++++++++++++++- 2 files changed, 172 insertions(+), 1 deletion(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 5d994a4545b..e79207df4fb 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -33,6 +33,7 @@ import { ReEmitter } from "../../src/ReEmitter"; import { SyncState } from "../../src/sync"; import { CallEvent, CallEventHandlerMap, MatrixCall } from "../../src/webrtc/call"; import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from "../../src/webrtc/callEventHandler"; +import { CallFeed } from "../../src/webrtc/callFeed"; import { GroupCallEventHandlerMap } from "../../src/webrtc/groupCall"; import { GroupCallEventHandlerEvent } from "../../src/webrtc/groupCallEventHandler"; import { IScreensharingOpts, MediaHandler } from "../../src/webrtc/mediaHandler"; @@ -384,6 +385,20 @@ export class MockCallMatrixClient extends TypedEventEmitter(), stream: new MockMediaStream("stream"), }; + public remoteUsermediaFeed: CallFeed; + public remoteScreensharingFeed: CallFeed; public reject = jest.fn(); public answerWithCallFeeds = jest.fn(); @@ -157,6 +160,14 @@ describe('Group Call', function() { groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); }); + it("does not initialize local call feed, if it already is", async () => { + await groupCall.initLocalCallFeed(); + jest.spyOn(groupCall, "initLocalCallFeed"); + await groupCall.enter(); + + expect(groupCall.initLocalCallFeed).not.toHaveBeenCalled(); + }); + it("sends state event to room when creating", async () => { await groupCall.create(); @@ -280,6 +291,119 @@ describe('Group Call', function() { } }); + describe("call feeds changing", () => { + let call: MockCall; + const currentFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("current")); + const newFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("new")); + + beforeEach(async () => { + jest.spyOn(currentFeed, "dispose"); + jest.spyOn(newFeed, "measureVolumeActivity"); + + jest.spyOn(groupCall, "emit"); + + call = new MockCall(room.roomId, groupCall.groupCallId); + + await groupCall.create(); + }); + + it("ignores changes, if we can't get user id of opponent", async () => { + const call = new MockCall(room.roomId, groupCall.groupCallId); + jest.spyOn(call, "getOpponentMember").mockReturnValue({ userId: undefined }); + + // @ts-ignore Mock + expect(() => groupCall.onCallFeedsChanged(call)).toThrowError(); + }); + + describe("usermedia feeds", () => { + it("adds new usermedia feed", async () => { + call.remoteUsermediaFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.userMediaFeeds).toStrictEqual([newFeed]); + }); + + it("replaces usermedia feed", async () => { + groupCall.userMediaFeeds = [currentFeed.typed()]; + + call.remoteUsermediaFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.userMediaFeeds).toStrictEqual([newFeed]); + }); + + it("removes usermedia feed", async () => { + groupCall.userMediaFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.userMediaFeeds).toHaveLength(0); + }); + }); + + describe("screenshare feeds", () => { + it("adds new screenshare feed", async () => { + call.remoteScreensharingFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.screenshareFeeds).toStrictEqual([newFeed]); + }); + + it("replaces screenshare feed", async () => { + groupCall.screenshareFeeds = [currentFeed.typed()]; + + call.remoteScreensharingFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.screenshareFeeds).toStrictEqual([newFeed]); + }); + + it("removes screenshare feed", async () => { + groupCall.screenshareFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.screenshareFeeds).toHaveLength(0); + }); + }); + + describe("feed replacing", () => { + it("replaces usermedia feed", async () => { + groupCall.userMediaFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.replaceUserMediaFeed(currentFeed, newFeed); + + const newFeeds = [newFeed]; + + expect(groupCall.userMediaFeeds).toStrictEqual(newFeeds); + expect(currentFeed.dispose).toHaveBeenCalled(); + expect(newFeed.measureVolumeActivity).toHaveBeenCalledWith(true); + expect(groupCall.emit).toHaveBeenCalledWith(GroupCallEvent.UserMediaFeedsChanged, newFeeds); + }); + + it("replaces screenshare feed", async () => { + groupCall.screenshareFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.replaceScreenshareFeed(currentFeed, newFeed); + + const newFeeds = [newFeed]; + + expect(groupCall.screenshareFeeds).toStrictEqual(newFeeds); + expect(currentFeed.dispose).toHaveBeenCalled(); + expect(newFeed.measureVolumeActivity).toHaveBeenCalledWith(true); + expect(groupCall.emit).toHaveBeenCalledWith(GroupCallEvent.ScreenshareFeedsChanged, newFeeds); + }); + }); + }); + describe("PTT calls", () => { beforeEach(async () => { // replace groupcall with a PTT one @@ -801,6 +925,19 @@ describe('Group Call', function() { expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled(); expect(groupCall.calls).toEqual([newMockCall]); }); + + it("starts to process incoming calls when we've entered", async () => { + // First we leave the call since we have already entered + groupCall.leave(); + + const call = new MockCall(room.roomId, groupCall.groupCallId); + mockClient.callEventHandler.calls = new Map([ + [call.callId, call.typed()], + ]); + await groupCall.enter(); + + expect(call.answerWithCallFeeds).toHaveBeenCalled(); + }); }); describe("screensharing", () => { @@ -885,9 +1022,28 @@ describe('Group Call', function() { ])); expect(groupCall.screenshareFeeds).toHaveLength(1); - expect(groupCall.getScreenshareFeedByUserId(call.invitee)).toBeDefined(); + expect(groupCall.getScreenshareFeedByUserId(call.invitee!)).toBeDefined(); + + groupCall.terminate(); + }); + + it("cleans up screensharing when terminating", async () => { + // @ts-ignore Mock + jest.spyOn(groupCall, "removeScreenshareFeed"); + jest.spyOn(mockClient.getMediaHandler(), "stopScreensharingStream"); + + await groupCall.setScreensharingEnabled(true); + + const screensharingFeed = groupCall.localScreenshareFeed; groupCall.terminate(); + + expect(mockClient.getMediaHandler()!.stopScreensharingStream).toHaveBeenCalledWith( + screensharingFeed!.stream, + ); + // @ts-ignore Mock + expect(groupCall.removeScreenshareFeed).toHaveBeenCalledWith(screensharingFeed); + expect(groupCall.localScreenshareFeed).toBeUndefined(); }); }); From 72b89fde6e9b072b0e131dfe393ee250269e5779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Tue, 20 Sep 2022 19:41:03 +0200 Subject: [PATCH 265/291] Add test for call transfers (#2677) --- spec/unit/webrtc/call.spec.ts | 93 +++++++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 4 deletions(-) diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 5504e3ae307..a470fcdd748 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -24,6 +24,7 @@ import { supportsMatrixCall, CallType, CallState, + CallParty, } from '../../../src/webrtc/call'; import { SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; import { @@ -42,20 +43,20 @@ import { Callback, EventType, IContent, ISendEventResponse, MatrixEvent, Room } const FAKE_ROOM_ID = "!foo:bar"; const CALL_LIFETIME = 60000; -const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { +const startVoiceCall = async (client: TestClient, call: MatrixCall, userId?: string): Promise => { const callPromise = call.placeVoiceCall(); await client.httpBackend.flush(""); await callPromise; - call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + call.getOpponentMember = jest.fn().mockReturnValue({ userId: userId ?? "@bob:bar.uk" }); }; -const startVideoCall = async (client: TestClient, call: MatrixCall): Promise => { +const startVideoCall = async (client: TestClient, call: MatrixCall, userId?: string): Promise => { const callPromise = call.placeVideoCall(); await client.httpBackend.flush(""); await callPromise; - call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + call.getOpponentMember = jest.fn().mockReturnValue({ userId: userId ?? "@bob:bar.uk" }); }; const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: string | number = "1") => { @@ -127,6 +128,7 @@ describe('Call', function() { }, } as unknown as Room; }; + client.client.getProfileInfo = jest.fn(); call = new MatrixCall({ client: client.client, @@ -1191,4 +1193,87 @@ describe('Call', function() { id: "usermedia_video_track", })); }); + + describe("call transfers", () => { + const ALICE_USER_ID = "@alice:foo"; + const ALICE_DISPLAY_NAME = "Alice"; + const ALICE_AVATAR_URL = "avatar.alice.foo"; + + const BOB_USER_ID = "@bob:foo"; + const BOB_DISPLAY_NAME = "Bob"; + const BOB_AVATAR_URL = "avatar.bob.foo"; + + beforeEach(() => { + mocked(client.client.getProfileInfo).mockImplementation(async (userId) => { + if (userId === ALICE_USER_ID) { + return { + displayname: ALICE_DISPLAY_NAME, + avatar_url: ALICE_AVATAR_URL, + }; + } else if (userId === BOB_USER_ID) { + return { + displayname: BOB_DISPLAY_NAME, + avatar_url: BOB_AVATAR_URL, + }; + } else { + return {}; + } + }); + }); + + it("transfers call to another call", async () => { + const newCall = new MatrixCall({ + client: client.client, + roomId: FAKE_ROOM_ID, + }); + + const callHangupListener = jest.fn(); + const newCallHangupListener = jest.fn(); + + call.on(CallEvent.Hangup, callHangupListener); + newCall.on(CallEvent.Error, () => { }); + newCall.on(CallEvent.Hangup, newCallHangupListener); + + await startVoiceCall(client, call, ALICE_USER_ID); + await startVoiceCall(client, newCall, BOB_USER_ID); + + await call.transferToCall(newCall); + + expect(mockSendEvent).toHaveBeenCalledWith(FAKE_ROOM_ID, EventType.CallReplaces, expect.objectContaining({ + target_user: { + id: ALICE_USER_ID, + display_name: ALICE_DISPLAY_NAME, + avatar_url: ALICE_AVATAR_URL, + }, + })); + expect(mockSendEvent).toHaveBeenCalledWith(FAKE_ROOM_ID, EventType.CallReplaces, expect.objectContaining({ + target_user: { + id: BOB_USER_ID, + display_name: BOB_DISPLAY_NAME, + avatar_url: BOB_AVATAR_URL, + }, + })); + + expect(callHangupListener).toHaveBeenCalledWith(call); + expect(newCallHangupListener).toHaveBeenCalledWith(newCall); + }); + + it("transfers a call to another user", async () => { + // @ts-ignore Mock + jest.spyOn(call, "terminate"); + + await startVoiceCall(client, call, ALICE_USER_ID); + await call.transfer(BOB_USER_ID); + + expect(mockSendEvent).toHaveBeenCalledWith(FAKE_ROOM_ID, EventType.CallReplaces, expect.objectContaining({ + target_user: { + id: BOB_USER_ID, + display_name: BOB_DISPLAY_NAME, + avatar_url: BOB_AVATAR_URL, + }, + })); + // @ts-ignore Mock + expect(call.terminate).toHaveBeenCalledWith(CallParty.Local, CallErrorCode.Transfered, true); + }); + }); }); From 72a6ec0dd3b96aea752924b163fe53de589d8680 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 22 Sep 2022 17:05:51 +0200 Subject: [PATCH 266/291] Add a few group call event handler tests (#2679) --- spec/test-utils/webrtc.ts | 22 ++- .../unit/webrtc/groupCallEventHandler.spec.ts | 168 +++++++++++++++++- 2 files changed, 178 insertions(+), 12 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index e79207df4fb..c190d22bff4 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -21,10 +21,12 @@ import { GroupCall, GroupCallIntent, GroupCallType, + IContent, ISendEventResponse, MatrixClient, MatrixEvent, Room, + RoomState, RoomStateEvent, RoomStateEventHandlerMap, } from "../../src"; @@ -381,8 +383,18 @@ export class MockCallMatrixClient extends TypedEventEmitter().mockReturnValue(SyncState.Syncing); getRooms = jest.fn().mockReturnValue([]); + getRoom = jest.fn(); typed(): MatrixClient { return this as unknown as MatrixClient; } + + emitRoomState(event: MatrixEvent, state: RoomState): void { + this.emit( + RoomStateEvent.Events, + event, + state, + null, + ); + } } export class MockCallFeed { @@ -436,15 +448,15 @@ export function installWebRTCMocks() { }; } -export function makeMockGroupCallStateEvent(roomId: string, groupCallId: string): MatrixEvent { +export function makeMockGroupCallStateEvent(roomId: string, groupCallId: string, content: IContent = { + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, +}): MatrixEvent { return { getType: jest.fn().mockReturnValue(EventType.GroupCallPrefix), getRoomId: jest.fn().mockReturnValue(roomId), getTs: jest.fn().mockReturnValue(0), - getContent: jest.fn().mockReturnValue({ - "m.type": GroupCallType.Video, - "m.intent": GroupCallIntent.Prompt, - }), + getContent: jest.fn().mockReturnValue(content), getStateKey: jest.fn().mockReturnValue(groupCallId), } as unknown as MatrixEvent; } diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts index cc9217f8dd1..e948f8afaaa 100644 --- a/spec/unit/webrtc/groupCallEventHandler.spec.ts +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -14,8 +14,21 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ClientEvent, GroupCall, Room, RoomState, RoomStateEvent } from "../../../src"; +import { mocked } from "jest-mock"; + +import { + ClientEvent, + GroupCall, + GroupCallIntent, + GroupCallState, + GroupCallType, + IContent, + MatrixEvent, + Room, + RoomState, +} from "../../../src"; import { SyncState } from "../../../src/sync"; +import { GroupCallTerminationReason } from "../../../src/webrtc/groupCall"; import { GroupCallEventHandler, GroupCallEventHandlerEvent } from "../../../src/webrtc/groupCallEventHandler"; import { flushPromises } from "../../test-utils/flushPromises"; import { @@ -53,6 +66,35 @@ describe('Group Call Event Handler', function() { (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); }); + describe("reacts to state changes", () => { + it("terminates call", async () => { + await groupCallEventHandler.start(); + mockClient.emitRoomState( + makeMockGroupCallStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID), + { roomId: FAKE_ROOM_ID } as unknown as RoomState, + ); + + const groupCall = groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID); + + expect(groupCall.state).toBe(GroupCallState.LocalCallFeedUninitialized); + + mockClient.emitRoomState( + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, { + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + "m.terminated": GroupCallTerminationReason.CallEnded, + }, + ), + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + ); + + expect(groupCall.state).toBe(GroupCallState.Ended); + }); + }); + it("waits until client starts syncing", async () => { mockClient.getSyncState.mockReturnValue(null); let isStarted = false; @@ -119,15 +161,13 @@ describe('Group Call Event Handler', function() { mockClient.on(GroupCallEventHandlerEvent.Incoming, onIncomingGroupCall); await groupCallEventHandler.start(); - mockClient.emit( - RoomStateEvent.Events, + mockClient.emitRoomState( makeMockGroupCallStateEvent( FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, ), { roomId: FAKE_ROOM_ID, } as unknown as RoomState, - null, ); expect(onIncomingGroupCall).toHaveBeenCalledWith(expect.objectContaining({ @@ -137,6 +177,40 @@ describe('Group Call Event Handler', function() { mockClient.off(GroupCallEventHandlerEvent.Incoming, onIncomingGroupCall); }); + it("handles data channel", async () => { + await groupCallEventHandler.start(); + + const dataChannelOptions = { + "maxPacketLifeTime": "life_time", + "maxRetransmits": "retransmits", + "ordered": "ordered", + "protocol": "protocol", + }; + + mockClient.emitRoomState( + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, + FAKE_GROUP_CALL_ID, + { + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + "dataChannelsEnabled": true, + dataChannelOptions, + }, + ), + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + ); + + // @ts-ignore Mock dataChannelsEnabled is private + expect(groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID)?.dataChannelsEnabled).toBe(true); + // @ts-ignore Mock dataChannelOptions is private + expect(groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID)?.dataChannelOptions).toStrictEqual( + dataChannelOptions, + ); + }); + it("sends member events to group calls", async () => { await groupCallEventHandler.start(); @@ -148,15 +222,95 @@ describe('Group Call Event Handler', function() { const mockStateEvent = makeMockGroupCallMemberStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID); - mockClient.emit( - RoomStateEvent.Events, + mockClient.emitRoomState( mockStateEvent, { roomId: FAKE_ROOM_ID, } as unknown as RoomState, - null, ); expect(mockGroupCall.onMemberStateChanged).toHaveBeenCalledWith(mockStateEvent); }); + + describe("ignoring invalid group call state events", () => { + let mockClientEmit: jest.Func; + + beforeEach(() => { + mockClientEmit = mockClient.emit = jest.fn(); + }); + + afterEach(() => { + groupCallEventHandler.stop(); + + jest.clearAllMocks(); + }); + + const setupCallAndStart = async (content?: IContent) => { + mocked(mockRoom.currentState.getStateEvents).mockReturnValue([ + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, + FAKE_GROUP_CALL_ID, + content, + ), + ] as unknown as MatrixEvent); + mockClient.getRooms.mockReturnValue([mockRoom]); + await groupCallEventHandler.start(); + }; + + it("ignores terminated calls", async () => { + await setupCallAndStart({ + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + "m.terminated": GroupCallTerminationReason.CallEnded, + }); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + + it("ignores calls with invalid type", async () => { + await setupCallAndStart({ + "m.type": "fake_type", + "m.intent": GroupCallIntent.Prompt, + }); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + + it("ignores calls with invalid intent", async () => { + await setupCallAndStart({ + "m.type": GroupCallType.Video, + "m.intent": "fake_intent", + }); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + + it("ignores calls without a room", async () => { + mockClient.getRoom.mockReturnValue(undefined); + + await setupCallAndStart(); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + }); }); From 2a0ffe12231b57f67dc7cd1ca18b8b4e7ad6cc15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Thu, 22 Sep 2022 17:06:01 +0200 Subject: [PATCH 267/291] Fix group call tests getting stuck (#2689) --- spec/unit/webrtc/groupCall.spec.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index a08c53f1d71..1c25087660e 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -161,11 +161,17 @@ describe('Group Call', function() { }); it("does not initialize local call feed, if it already is", async () => { + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + await groupCall.initLocalCallFeed(); jest.spyOn(groupCall, "initLocalCallFeed"); await groupCall.enter(); expect(groupCall.initLocalCallFeed).not.toHaveBeenCalled(); + + groupCall.leave(); }); it("sends state event to room when creating", async () => { From a2981efac3921763e9c909bf4aef1b904ed55c73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 23 Sep 2022 18:33:31 +0200 Subject: [PATCH 268/291] Add `MatrixClient` group call tests (#2692) Co-authored-by: Robin --- spec/unit/matrix-client.spec.ts | 35 ++++++++++++ spec/unit/webrtc/groupCall.spec.ts | 89 ++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+) diff --git a/spec/unit/matrix-client.spec.ts b/spec/unit/matrix-client.spec.ts index 2b8faf5065a..98c496d76cb 100644 --- a/spec/unit/matrix-client.spec.ts +++ b/spec/unit/matrix-client.spec.ts @@ -1714,4 +1714,39 @@ describe("MatrixClient", function() { expect(newSourceRoom._state.get(PolicyScope.User)?.[eventId]).toBeFalsy(); }); }); + + describe("using E2EE in group calls", () => { + const opts = { + baseUrl: "https://my.home.server", + idBaseUrl: identityServerUrl, + accessToken: "my.access.token", + store: store, + scheduler: scheduler, + userId: userId, + }; + + it("enables E2EE by default", () => { + const client = new MatrixClient(opts); + + expect(client.getUseE2eForGroupCall()).toBe(true); + }); + + it("enables E2EE when enabled explicitly", () => { + const client = new MatrixClient({ + useE2eForGroupCall: true, + ...opts, + }); + + expect(client.getUseE2eForGroupCall()).toBe(true); + }); + + it("disables E2EE if disabled explicitly", () => { + const client = new MatrixClient({ + useE2eForGroupCall: false, + ...opts, + }); + + expect(client.getUseE2eForGroupCall()).toBe(false); + }); + }); }); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 1c25087660e..cfc70cfa4f8 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -1118,4 +1118,93 @@ describe('Group Call', function() { expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_3); }); }); + + describe("creating group calls", () => { + let client: MatrixClient; + + beforeEach(() => { + client = new MatrixClient({ + baseUrl: "base_url", + request: (() => {}) as any, // NOP + }); + + jest.spyOn(client, "sendStateEvent").mockResolvedValue({} as any); + }); + + afterEach(() => { + client.stopClient(); + }); + + it("throws when there already is a call", async () => { + jest.spyOn(client, "getRoom").mockReturnValue(new Room("room_id", client, "my_user_id")); + + await client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + ); + + await expect(client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + )).rejects.toThrow("room_id already has an existing group call"); + }); + + it("throws if the room doesn't exist", async () => { + await expect(client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + )).rejects.toThrow("Cannot find room room_id"); + }); + + describe("correctly passes parameters", () => { + beforeEach(() => { + jest.spyOn(client, "getRoom").mockReturnValue(new Room("room_id", client, "my_user_id")); + }); + + it("correctly passes voice ptt room call", async () => { + const groupCall = await client.createGroupCall( + "room_id", + GroupCallType.Voice, + true, + GroupCallIntent.Room, + ); + + expect(groupCall.type).toBe(GroupCallType.Voice); + expect(groupCall.isPtt).toBe(true); + expect(groupCall.intent).toBe(GroupCallIntent.Room); + }); + + it("correctly passes voice ringing call", async () => { + const groupCall = await client.createGroupCall( + "room_id", + GroupCallType.Voice, + false, + GroupCallIntent.Ring, + ); + + expect(groupCall.type).toBe(GroupCallType.Voice); + expect(groupCall.isPtt).toBe(false); + expect(groupCall.intent).toBe(GroupCallIntent.Ring); + }); + + it("correctly passes video prompt call", async () => { + const groupCall = await client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + ); + + expect(groupCall.type).toBe(GroupCallType.Video); + expect(groupCall.isPtt).toBe(false); + expect(groupCall.intent).toBe(GroupCallIntent.Prompt); + }); + }); + }); }); From af6f9d49f486507d52ba24ccedc204029ee62752 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 26 Sep 2022 12:02:19 +0200 Subject: [PATCH 269/291] Add `CallEventHandler` tests (#2696) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add `CallEventHandler` tests Signed-off-by: Šimon Brandner * Avoid tests hanging Signed-off-by: Šimon Brandner Signed-off-by: Šimon Brandner --- spec/unit/webrtc/callEventHandler.spec.ts | 130 ++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/spec/unit/webrtc/callEventHandler.spec.ts b/spec/unit/webrtc/callEventHandler.spec.ts index 1fe6f788f32..5bed6cea3df 100644 --- a/spec/unit/webrtc/callEventHandler.spec.ts +++ b/spec/unit/webrtc/callEventHandler.spec.ts @@ -20,24 +20,33 @@ import { EventTimeline, EventTimelineSet, EventType, + GroupCallIntent, + GroupCallType, IRoomTimelineData, + MatrixCall, MatrixEvent, Room, RoomEvent, + RoomMember, } from "../../../src"; import { MatrixClient } from "../../../src/client"; import { CallEventHandler, CallEventHandlerEvent } from "../../../src/webrtc/callEventHandler"; import { GroupCallEventHandler } from "../../../src/webrtc/groupCallEventHandler"; import { SyncState } from "../../../src/sync"; +import { installWebRTCMocks, MockRTCPeerConnection } from "../../test-utils/webrtc"; +import { sleep } from "../../../src/utils"; describe("CallEventHandler", () => { let client: MatrixClient; beforeEach(() => { + installWebRTCMocks(); + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}).client; client.callEventHandler = new CallEventHandler(client); client.callEventHandler.start(); client.groupCallEventHandler = new GroupCallEventHandler(client); client.groupCallEventHandler.start(); + client.sendStateEvent = jest.fn().mockResolvedValue({}); }); afterEach(() => { @@ -45,6 +54,14 @@ describe("CallEventHandler", () => { client.groupCallEventHandler.stop(); }); + const sync = async () => { + client.getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); + client.emit(ClientEvent.Sync, SyncState.Syncing); + + // We can't await the event processing + await sleep(10); + }; + it("should enforce inbound toDevice message ordering", async () => { const callEventHandler = client.callEventHandler; const event1 = new MatrixEvent({ @@ -140,4 +157,117 @@ describe("CallEventHandler", () => { expect(incomingCallEmitted).not.toHaveBeenCalled(); }); + + it("should ignore non-call events", async () => { + // @ts-ignore Mock handleCallEvent is private + jest.spyOn(client.callEventHandler, "handleCallEvent"); + jest.spyOn(client, "checkTurnServers").mockReturnValue(undefined); + + const room = new Room("!room:id", client, "@user:id"); + const timelineData: IRoomTimelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; + + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.RoomMessage, + room_id: "!room:id", + content: { + text: "hello", + + }, + }), room, false, false, timelineData); + await sync(); + + // @ts-ignore Mock handleCallEvent is private + expect(client.callEventHandler.handleCallEvent).not.toHaveBeenCalled(); + }); + + describe("handleCallEvent()", () => { + const incomingCallListener = jest.fn(); + let timelineData: IRoomTimelineData; + let room: Room; + + beforeEach(() => { + room = new Room("!room:id", client, client.getUserId()); + timelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; + + jest.spyOn(client, "checkTurnServers").mockReturnValue(undefined); + jest.spyOn(client, "getRoom").mockReturnValue(room); + jest.spyOn(room, "getMember").mockReturnValue({ user_id: client.getUserId() } as unknown as RoomMember); + + client.on(CallEventHandlerEvent.Incoming, incomingCallListener); + }); + + afterEach(() => { + MockRTCPeerConnection.resetInstances(); + jest.resetAllMocks(); + }); + + it("should create a call when receiving an invite", async () => { + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.CallInvite, + room_id: "!room:id", + content: { + call_id: "123", + }, + }), room, false, false, timelineData); + await sync(); + + expect(incomingCallListener).toHaveBeenCalled(); + }); + + it("should handle group call event", async () => { + let call: MatrixCall; + const groupCall = await client.createGroupCall( + room.roomId, + GroupCallType.Voice, + false, + GroupCallIntent.Ring, + ); + const SESSION_ID = "sender_session_id"; + const GROUP_CALL_ID = "group_call_id"; + const DEVICE_ID = "device_id"; + + incomingCallListener.mockImplementation((c) => call = c); + jest.spyOn(client.groupCallEventHandler, "getGroupCallById").mockReturnValue(groupCall); + // @ts-ignore Mock onIncomingCall is private + jest.spyOn(groupCall, "onIncomingCall"); + + await groupCall.enter(); + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.CallInvite, + room_id: "!room:id", + content: { + call_id: "123", + conf_id: GROUP_CALL_ID, + device_id: DEVICE_ID, + sender_session_id: SESSION_ID, + dest_session_id: client.getSessionId(), + }, + }), room, false, false, timelineData); + await sync(); + + expect(incomingCallListener).toHaveBeenCalled(); + expect(call.groupCallId).toBe(GROUP_CALL_ID); + // @ts-ignore Mock opponentDeviceId is private + expect(call.opponentDeviceId).toBe(DEVICE_ID); + expect(call.getOpponentSessionId()).toBe(SESSION_ID); + // @ts-ignore Mock onIncomingCall is private + expect(groupCall.onIncomingCall).toHaveBeenCalledWith(call); + + groupCall.terminate(false); + }); + + it("ignores a call with a different invitee than us", async () => { + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.CallInvite, + room_id: "!room:id", + content: { + call_id: "123", + invitee: "@bob:bar", + }, + }), room, false, false, timelineData); + await sync(); + + expect(incomingCallListener).not.toHaveBeenCalled(); + }); + }); }); From ab39ee37d61dbfc44b5de2269a907c466170a4c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 26 Sep 2022 12:02:41 +0200 Subject: [PATCH 270/291] Add more `MatrixCall` tests (#2697) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner Signed-off-by: Šimon Brandner --- spec/test-utils/webrtc.ts | 3 + spec/unit/webrtc/call.spec.ts | 124 +++++++++++++++++++++++++++++++++- 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index c190d22bff4..09849d4f32b 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -103,6 +103,7 @@ export class MockRTCPeerConnection { private negotiationNeededListener: () => void; public iceCandidateListener?: (e: RTCPeerConnectionIceEvent) => void; + public onTrackListener?: (e: RTCTrackEvent) => void; private needsNegotiation = false; public readyToNegotiate: Promise; private onReadyToNegotiate: () => void; @@ -143,6 +144,8 @@ export class MockRTCPeerConnection { this.negotiationNeededListener = listener; } else if (type == 'icecandidate') { this.iceCandidateListener = listener; + } else if (type == 'track') { + this.onTrackListener = listener; } } createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index a470fcdd748..5d29fe15adf 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -26,7 +26,13 @@ import { CallState, CallParty, } from '../../../src/webrtc/call'; -import { SDPStreamMetadata, SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; +import { + MCallAnswer, + MCallHangupReject, + SDPStreamMetadata, + SDPStreamMetadataKey, + SDPStreamMetadataPurpose, +} from '../../../src/webrtc/callEventTypes'; import { DUMMY_SDP, MockMediaHandler, @@ -102,6 +108,8 @@ describe('Call', function() { // We retain a reference to this in the correct Mock type let mockSendEvent: jest.Mock, [string, string, IContent, string, Callback]>; + const errorListener = () => {}; + beforeEach(function() { prevNavigator = global.navigator; prevDocument = global.document; @@ -135,7 +143,7 @@ describe('Call', function() { roomId: FAKE_ROOM_ID, }); // call checks one of these is wired up - call.on(CallEvent.Error, () => {}); + call.on(CallEvent.Error, errorListener); }); afterEach(function() { @@ -1276,4 +1284,116 @@ describe('Call', function() { expect(call.terminate).toHaveBeenCalledWith(CallParty.Local, CallErrorCode.Transfered, true); }); }); + + describe("onTrack", () => { + it("ignores streamless track", async () => { + // @ts-ignore Mock pushRemoteFeed() is private + jest.spyOn(call, "pushRemoteFeed"); + + await call.placeVoiceCall(); + + (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener( + { streams: [], track: new MockMediaStreamTrack("track_ev", "audio") } as unknown as RTCTrackEvent, + ); + + // @ts-ignore Mock pushRemoteFeed() is private + expect(call.pushRemoteFeed).not.toHaveBeenCalled(); + }); + + it("correctly pushes", async () => { + // @ts-ignore Mock pushRemoteFeed() is private + jest.spyOn(call, "pushRemoteFeed"); + + await call.placeVoiceCall(); + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + answer: { + sdp: DUMMY_SDP, + }, + })); + + const stream = new MockMediaStream("stream_ev", [new MockMediaStreamTrack("track_ev", "audio")]); + (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener( + { streams: [stream], track: stream.getAudioTracks()[0] } as unknown as RTCTrackEvent, + ); + + // @ts-ignore Mock pushRemoteFeed() is private + expect(call.pushRemoteFeed).toHaveBeenCalledWith(stream); + // @ts-ignore Mock pushRemoteFeed() is private + expect(call.removeTrackListeners.has(stream)).toBe(true); + }); + }); + + describe("onHangupReceived()", () => { + it("ends call on onHangupReceived() if state is ringing", async () => { + expect(call.callHasEnded()).toBe(false); + + call.state = CallState.Ringing; + call.onHangupReceived({} as MCallHangupReject); + + expect(call.callHasEnded()).toBe(true); + }); + + it("ends call on onHangupReceived() if party id matches", async () => { + expect(call.callHasEnded()).toBe(false); + + await call.initWithInvite({ + getContent: jest.fn().mockReturnValue({ + version: "1", + call_id: "call_id", + party_id: "remote_party_id", + lifetime: CALL_LIFETIME, + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + } as unknown as MatrixEvent); + call.onHangupReceived({ version: "1", party_id: "remote_party_id" } as MCallHangupReject); + + expect(call.callHasEnded()).toBe(true); + }); + }); + + it.each( + Object.values(CallState), + )("ends call on onRejectReceived() if in correct state (state=%s)", async (state: CallState) => { + expect(call.callHasEnded()).toBe(false); + + call.state = state; + call.onRejectReceived({} as MCallHangupReject); + + expect(call.callHasEnded()).toBe( + [CallState.InviteSent, CallState.Ringing, CallState.Ended].includes(state), + ); + }); + + it("terminates call when answered elsewhere", async () => { + await call.placeVoiceCall(); + + expect(call.callHasEnded()).toBe(false); + + call.onAnsweredElsewhere({} as MCallAnswer); + + expect(call.callHasEnded()).toBe(true); + }); + + it("throws when there is no error listener", async () => { + call.off(CallEvent.Error, errorListener); + + expect(call.placeVoiceCall()).rejects.toThrow(); + }); + + describe("hasPeerConnection()", () => { + it("hasPeerConnection() returns false if there is no peer connection", () => { + expect(call.hasPeerConnection).toBe(false); + }); + + it("hasPeerConnection() returns true if there is a peer connection", async () => { + await call.placeVoiceCall(); + expect(call.hasPeerConnection).toBe(true); + }); + }); }); From e48d919cd4e0538e4d7527a8ce7c93b662aebb6d Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 27 Sep 2022 17:25:04 +0100 Subject: [PATCH 271/291] Fix ICE restarts (#2702) We didn't reset the 'seen end of candidates' flag when doign an ICE restart, so we would have ignored all locally gathered candidates on an ICE restart. --- src/webrtc/call.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 4b5dcbae614..7f60a863eeb 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2025,6 +2025,7 @@ export class MatrixCall extends TypedEventEmitter Date: Thu, 13 Oct 2022 11:56:46 -0400 Subject: [PATCH 272/291] Don't block muting/unmuting on network requests (PTT mode will still block on them, as expected) --- src/webrtc/groupCall.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index bf8022c2b9d..350215b9a0f 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -505,6 +505,8 @@ export class GroupCall extends TypedEventEmitter< setTracksEnabled(call.localUsermediaFeed.stream.getAudioTracks(), !muted); } + this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); + if (!sendUpdatesBefore) { try { await Promise.all(this.calls.map(c => c.sendMetadataUpdate())); @@ -513,7 +515,6 @@ export class GroupCall extends TypedEventEmitter< } } - this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); return true; } From dfe535bc07e907cbf417a69ff80848e6228ae91a Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 17 Oct 2022 20:14:44 +0100 Subject: [PATCH 273/291] More debugging for multiple group calls (#2766) --- src/webrtc/groupCallEventHandler.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index ddfc806e389..e38d7f2b419 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -125,6 +125,11 @@ export class GroupCallEventHandler { continue; } + logger.debug( + `Choosing group call ${callEvent.getStateKey()} with TS ` + + `${callEvent.getTs()} for room ${room.roomId} from ${callEvents.length} possible calls.`, + ); + this.createGroupCallFromRoomStateEvent(callEvent); break; } From c57c8978cf25b0ae846197052c17181f6bb7624a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Oct 2022 16:00:54 +0100 Subject: [PATCH 274/291] Fix screenshare failing after several attempts (#2771) * Fix screenshare failing after several attempts Re-use any existing transceivers when screen sharing. This prevents transceivers accumulating and making the SDP too big: see linked bug. This also switches from `addTrack()` to `addTransceiver ()` which is not that large of a change, other than having to explicitly find the transceivers after an offer has arrived rather than just adding tracks and letting WebRTC take care of it. Fixes https://github.com/vector-im/element-call/issues/625 * Fix tests * Unused import * Use a map instead of an array * Add comment * more comment * Remove commented code * Remove unintentional debugging * Add test for screenshare transceiver re-use * Type alias for transceiver map --- spec/test-utils/webrtc.ts | 44 +++++++-- spec/unit/webrtc/call.spec.ts | 122 ++++++++++++++++-------- src/webrtc/call.ts | 173 ++++++++++++++++++++++------------ src/webrtc/groupCall.ts | 4 +- 4 files changed, 235 insertions(+), 108 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 09849d4f32b..ac9148212b9 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -104,12 +104,12 @@ export class MockRTCPeerConnection { private negotiationNeededListener: () => void; public iceCandidateListener?: (e: RTCPeerConnectionIceEvent) => void; public onTrackListener?: (e: RTCTrackEvent) => void; - private needsNegotiation = false; + public needsNegotiation = false; public readyToNegotiate: Promise; private onReadyToNegotiate: () => void; localDescription: RTCSessionDescription; signalingState: RTCSignalingState = "stable"; - public senders: MockRTCRtpSender[] = []; + public transceivers: MockRTCRtpTransceiver[] = []; public static triggerAllNegotiations(): void { for (const inst of this.instances) { @@ -169,12 +169,23 @@ export class MockRTCPeerConnection { } close() { } getStats() { return []; } - addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { + addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { this.needsNegotiation = true; this.onReadyToNegotiate(); + const newSender = new MockRTCRtpSender(track); - this.senders.push(newSender); - return newSender; + const newReceiver = new MockRTCRtpReceiver(track); + + const newTransceiver = new MockRTCRtpTransceiver(this); + newTransceiver.sender = newSender as unknown as RTCRtpSender; + newTransceiver.receiver = newReceiver as unknown as RTCRtpReceiver; + + this.transceivers.push(newTransceiver); + + return newTransceiver; + } + addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { + return this.addTransceiver(track).sender as unknown as MockRTCRtpSender; } removeTrack() { @@ -182,9 +193,8 @@ export class MockRTCPeerConnection { this.onReadyToNegotiate(); } - getSenders(): MockRTCRtpSender[] { return this.senders; } - - getTransceivers = jest.fn().mockReturnValue([]); + getTransceivers(): MockRTCRtpTransceiver[] { return this.transceivers; } + getSenders(): MockRTCRtpSender[] { return this.transceivers.map(t => t.sender as unknown as MockRTCRtpSender); } doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { @@ -198,7 +208,23 @@ export class MockRTCRtpSender { constructor(public track: MockMediaStreamTrack) { } replaceTrack(track: MockMediaStreamTrack) { this.track = track; } - setCodecPreferences(prefs: RTCRtpCodecCapability[]): void {} +} + +export class MockRTCRtpReceiver { + constructor(public track: MockMediaStreamTrack) { } +} + +export class MockRTCRtpTransceiver { + constructor(private peerConn: MockRTCPeerConnection) {} + + public sender: RTCRtpSender; + public receiver: RTCRtpReceiver; + + public set direction(_: string) { + this.peerConn.needsNegotiation = true; + } + + setCodecPreferences = jest.fn(); } export class MockMediaStreamTrack { diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index e592cba9b67..df9c2aee09b 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -41,7 +41,6 @@ import { installWebRTCMocks, MockRTCPeerConnection, SCREENSHARE_STREAM_ID, - MockRTCRtpSender, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; import { EventType, IContent, ISendEventResponse, MatrixEvent, Room } from "../../../src"; @@ -370,17 +369,15 @@ describe('Call', function() { ).typed(), ); - const usermediaSenders: Array = (call as any).usermediaSenders; + // XXX: Lots of inspecting the prvate state of the call object here + const transceivers: Map = (call as any).transceivers; expect(call.localUsermediaStream.id).toBe("stream"); expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("new_audio_track"); expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); - expect(usermediaSenders.find((sender) => { - return sender?.track?.kind === "audio"; - }).track.id).toBe("new_audio_track"); - expect(usermediaSenders.find((sender) => { - return sender?.track?.kind === "video"; - }).track.id).toBe("video_track"); + // call has a function for generating these but we hardcode here to avoid exporting it + expect(transceivers.get("m.usermedia:audio").sender.track.id).toBe("new_audio_track"); + expect(transceivers.get("m.usermedia:video").sender.track.id).toBe("video_track"); }); it("should handle upgrade to video call", async () => { @@ -400,16 +397,13 @@ describe('Call', function() { // setLocalVideoMuted probably? await (call as any).upgradeCall(false, true); - const usermediaSenders: Array = (call as any).usermediaSenders; + // XXX: More inspecting private state of the call object + const transceivers: Map = (call as any).transceivers; expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("usermedia_audio_track"); expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("usermedia_video_track"); - expect(usermediaSenders.find((sender) => { - return sender?.track?.kind === "audio"; - }).track.id).toBe("usermedia_audio_track"); - expect(usermediaSenders.find((sender) => { - return sender?.track?.kind === "video"; - }).track.id).toBe("usermedia_video_track"); + expect(transceivers.get("m.usermedia:audio").sender.track.id).toBe("usermedia_audio_track"); + expect(transceivers.get("m.usermedia:video").sender.track.id).toBe("usermedia_video_track"); }); it("should handle SDPStreamMetadata changes", async () => { @@ -479,6 +473,23 @@ describe('Call', function() { }); describe("should deduce the call type correctly", () => { + beforeEach(async () => { + // start an incoming call, but add no feeds + await call.initWithInvite({ + getContent: jest.fn().mockReturnValue({ + version: "1", + call_id: "call_id", + party_id: "remote_party_id", + lifetime: CALL_LIFETIME, + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + getLocalAge: () => 1, + } as unknown as MatrixEvent); + }); + it("if no video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); @@ -1057,9 +1068,24 @@ describe('Call', function() { }); describe("Screen sharing", () => { + const waitNegotiateFunc = resolve => { + mockSendEvent.mockImplementationOnce(() => { + // Note that the peer connection here is a dummy one and always returns + // dummy SDP, so there's not much point returning the content: the SDP will + // always be the same. + resolve(); + return Promise.resolve({ event_id: "foo" }); + }); + }; + beforeEach(async () => { await startVoiceCall(client, call); + const sendNegotiatePromise = new Promise(waitNegotiateFunc); + + MockRTCPeerConnection.triggerAllNegotiations(); + await sendNegotiatePromise; + await call.onAnswerReceived(makeMockEvent("@test:foo", { "version": 1, "call_id": call.callId, @@ -1090,12 +1116,7 @@ describe('Call', function() { ).toHaveLength(1); mockSendEvent.mockReset(); - const sendNegotiatePromise = new Promise(resolve => { - mockSendEvent.mockImplementationOnce(() => { - resolve(); - return Promise.resolve({ event_id: "foo" }); - }); - }); + const sendNegotiatePromise = new Promise(waitNegotiateFunc); MockRTCPeerConnection.triggerAllNegotiations(); await sendNegotiatePromise; @@ -1130,29 +1151,52 @@ describe('Call', function() { headerExtensions: [], }); - const prom = new Promise(resolve => { - const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; - mockPeerConn.addTrack = jest.fn().mockImplementation((track: MockMediaStreamTrack) => { - const mockSender = new MockRTCRtpSender(track); - mockPeerConn.getTransceivers.mockReturnValue([{ - sender: mockSender, - setCodecPreferences: (prefs: RTCRtpCodecCapability[]) => { - expect(prefs).toEqual([ - expect.objectContaining({ mimeType: "video/somethingelse" }), - ]); - - resolve(); - }, - }]); + mockSendEvent.mockReset(); + const sendNegotiatePromise = new Promise(waitNegotiateFunc); - return mockSender; - }); - }); + await call.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + + await sendNegotiatePromise; + + const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + expect( + mockPeerConn.transceivers[mockPeerConn.transceivers.length - 1].setCodecPreferences, + ).toHaveBeenCalledWith([expect.objectContaining({ mimeType: "video/somethingelse" })]); + }); + + it("re-uses transceiver when screen sharing is re-enabled", async () => { + const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + + // sanity check: we should start with one transciever (user media audio) + expect(mockPeerConn.transceivers.length).toEqual(1); + + const screenshareOnProm1 = new Promise(waitNegotiateFunc); + + await call.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + + await screenshareOnProm1; + + // we should now have another transciever for the screenshare + expect(mockPeerConn.transceivers.length).toEqual(2); + + const screenshareOffProm = new Promise(waitNegotiateFunc); + await call.setScreensharingEnabled(false); + MockRTCPeerConnection.triggerAllNegotiations(); + await screenshareOffProm; + + // both transceivers should still be there + expect(mockPeerConn.transceivers.length).toEqual(2); + const screenshareOnProm2 = new Promise(waitNegotiateFunc); await call.setScreensharingEnabled(true); MockRTCPeerConnection.triggerAllNegotiations(); + await screenshareOnProm2; - await prom; + // should still be two, ie. another one should not have been created + // when re-enabling the screen share. + expect(mockPeerConn.transceivers.length).toEqual(2); }); }); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index ce1cae8cd08..1fb6a4c2293 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -308,6 +308,16 @@ export type CallEventHandlerMap = { [CallEvent.SendVoipEvent]: (event: Record) => void; }; +// The key of the transceiver map (purpose + media type, separated by ':') +type TransceiverKey = string; + +// generates keys for the map of transceivers +// kind is unfortunately a string rather than MediaType as this is the type of +// track.kind +function getTransceiverKey(purpose: SDPStreamMetadataPurpose, kind: TransceiverKey): string { + return purpose + ':' + kind; +} + /** * Construct a new Matrix Call. * @constructor @@ -345,8 +355,10 @@ export class MatrixCall extends TypedEventEmitter = []; - private usermediaSenders: Array = []; - private screensharingSenders: Array = []; + + // our transceivers for each purpose and type of media + private transceivers = new Map(); + private inviteOrAnswerSent = false; private waitForLocalAVStream: boolean; private successor: MatrixCall; @@ -634,6 +646,18 @@ export class MatrixCall extends TypedEventEmitter t.receiver.track == track); + this.transceivers.set(getTransceiverKey(purpose, track.kind), transceiver); + } + } + this.emit(CallEvent.FeedsChanged, this.feeds); logger.info( @@ -675,6 +699,12 @@ export class MatrixCall extends TypedEventEmitter t.receiver.track == track); + this.transceivers.set(getTransceiverKey(purpose, track.kind), transceiver); + } + this.emit(CallEvent.FeedsChanged, this.feeds); logger.info(`Call ${this.callId} pushed remote stream (id="${stream.id}", active="${stream.active}")`); @@ -722,11 +752,6 @@ export class MatrixCall extends TypedEventEmitter { return track.kind === "video"; }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); + + const sender = this.transceivers.get(getTransceiverKey( + SDPStreamMetadataPurpose.Usermedia, "video", + )).sender; + sender.replaceTrack(track); this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); @@ -1183,9 +1243,9 @@ export class MatrixCall extends TypedEventEmitter { return track.kind === "video"; }); - const sender = this.usermediaSenders.find((sender) => { - return sender.track?.kind === "video"; - }); + const sender = this.transceivers.get(getTransceiverKey( + SDPStreamMetadataPurpose.Usermedia, "video", + )).sender; sender.replaceTrack(track); this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream); @@ -1219,28 +1279,30 @@ export class MatrixCall extends TypedEventEmitter { - return sender.track?.kind === track.kind; - }); + const tKey = getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, track.kind); - let newSender: RTCRtpSender; + const oldSender = this.transceivers.get(tKey)?.sender; + let added = false; + if (oldSender) { + try { + logger.info( + `Call ${this.callId} `+ + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + added = true; + } catch (error) { + logger.warn(`replaceTrack failed: adding new transceiver instead`, error); + } + } - try { - logger.info( - `Call ${this.callId} `+ - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } catch (error) { + if (!added) { logger.info( `Call ${this.callId} `+ `Adding track (` + @@ -1250,13 +1312,13 @@ export class MatrixCall extends TypedEventEmitter => { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 350215b9a0f..c20592f52f9 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -607,7 +607,9 @@ export class GroupCall extends TypedEventEmitter< return false; } } else { - await Promise.all(this.calls.map(call => call.removeLocalFeed(call.localScreensharingFeed))); + await Promise.all(this.calls.map(call => { + if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); + })); this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); this.removeScreenshareFeed(this.localScreenshareFeed); this.localScreenshareFeed = undefined; From 5a0787349d4951012eabe72f3363c17bdcda0d56 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 19 Oct 2022 18:03:12 +0100 Subject: [PATCH 275/291] Fix connectivity regressions (#2780) * Fix connectivity regressions Switches back to addTrack, digging the transceivers out manually to re-use, because the only way to group tracks into streams re-using trasceivers from the offer is to use setStreams which FF doesn't implement. * Remove comments --- src/webrtc/call.ts | 39 ++++++++++++++------------------------- 1 file changed, 14 insertions(+), 25 deletions(-) diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 1fb6a4c2293..c3967dba3ca 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -647,17 +647,6 @@ export class MatrixCall extends TypedEventEmitter t.receiver.track == track); - this.transceivers.set(getTransceiverKey(purpose, track.kind), transceiver); - } - } - this.emit(CallEvent.FeedsChanged, this.feeds); logger.info( @@ -700,11 +689,6 @@ export class MatrixCall extends TypedEventEmitter t.receiver.track == track); - this.transceivers.set(getTransceiverKey(purpose, track.kind), transceiver); - } - this.emit(CallEvent.FeedsChanged, this.feeds); logger.info(`Call ${this.callId} pushed remote stream (id="${stream.id}", active="${stream.active}")`); @@ -773,16 +757,23 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender)); } } } @@ -1313,10 +1304,8 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender)); } } } From eddd0cafe83509f2fe464cfbe957dda49ee83040 Mon Sep 17 00:00:00 2001 From: David Baker Date: Mon, 24 Oct 2022 10:20:04 +0100 Subject: [PATCH 276/291] Add throwOnFail to groupCall.setScreensharingEnabled (#2787) * Add throwOnFail to groupCall.setScreensharingEnabled For https://github.com/vector-im/element-call/pull/652 * Update mediaHandler.ts --- src/webrtc/groupCall.ts | 1 + src/webrtc/mediaHandler.ts | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c20592f52f9..eb122a395ce 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -600,6 +600,7 @@ export class GroupCall extends TypedEventEmitter< return true; } catch (error) { + if (opts.throwOnFail) throw error; logger.error("Enabling screensharing error", error); this.emit(GroupCallEvent.Error, new GroupCallError(GroupCallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", error), diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 2eba5f2f534..718adfd4f60 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -33,6 +33,11 @@ export type MediaHandlerEventHandlerMap = { export interface IScreensharingOpts { desktopCapturerSourceId?: string; audio?: boolean; + // For electron screen capture, there are very few options for detecting electron + // apart from inspecting the user agent or just trying getDisplayMedia() and + // catching the failure, so we do the latter - this flag tells the function to just + // throw an error so we can catch it in this case, rather than logging and emitting. + throwOnFail?: boolean; } export class MediaHandler extends TypedEventEmitter< From 8cd5aac128d7d715d5daf3b7b63b756651f41461 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 24 Oct 2022 12:11:52 -0400 Subject: [PATCH 277/291] Add event and message capabilities to RoomWidgetClient --- spec/unit/embedded.spec.ts | 67 +++++++++++++++++++++++++++- src/client.ts | 5 +-- src/embedded.ts | 90 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 154 insertions(+), 8 deletions(-) diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index f704e847d13..3a144215c6a 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -30,7 +30,7 @@ import { ITurnServer, } from "matrix-widget-api"; -import { createRoomWidgetClient } from "../../src/matrix"; +import { createRoomWidgetClient, MsgType } from "../../src/matrix"; import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client"; import { SyncState } from "../../src/sync"; import { ICapabilities } from "../../src/embedded"; @@ -43,10 +43,15 @@ class MockWidgetApi extends EventEmitter { public requestCapability = jest.fn(); public requestCapabilities = jest.fn(); public requestCapabilityForRoomTimeline = jest.fn(); + public requestCapabilityToSendEvent = jest.fn(); + public requestCapabilityToReceiveEvent = jest.fn(); + public requestCapabilityToSendMessage = jest.fn(); + public requestCapabilityToReceiveMessage = jest.fn(); public requestCapabilityToSendState = jest.fn(); public requestCapabilityToReceiveState = jest.fn(); public requestCapabilityToSendToDevice = jest.fn(); public requestCapabilityToReceiveToDevice = jest.fn(); + public sendRoomEvent = jest.fn(() => ({ event_id: `$${Math.random()}` })); public sendStateEvent = jest.fn(); public sendToDevice = jest.fn(); public readStateEvents = jest.fn(() => []); @@ -75,6 +80,66 @@ describe("RoomWidgetClient", () => { await client.startClient(); }; + describe("events", () => { + it("sends", async () => { + await makeClient({ sendEvent: ["org.matrix.rageshake_request"] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendEvent).toHaveBeenCalledWith("org.matrix.rageshake_request"); + await client.sendEvent("!1:example.org", "org.matrix.rageshake_request", { request_id: 123 }); + expect(widgetApi.sendRoomEvent).toHaveBeenCalledWith( + "org.matrix.rageshake_request", { request_id: 123 }, "!1:example.org", + ); + }); + + it("receives", async () => { + const event = new MatrixEvent({ + type: "org.matrix.rageshake_request", + event_id: "$pduhfiidph", + room_id: "!1:example.org", + sender: "@alice:example.org", + content: { request_id: 123 }, + }).getEffectiveEvent(); + + await makeClient({ receiveEvent: ["org.matrix.rageshake_request"] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToReceiveEvent).toHaveBeenCalledWith("org.matrix.rageshake_request"); + + const emittedEvent = new Promise(resolve => client.once(ClientEvent.Event, resolve)); + const emittedSync = new Promise(resolve => client.once(ClientEvent.Sync, resolve)); + widgetApi.emit( + `action:${WidgetApiToWidgetAction.SendEvent}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.SendEvent}`, { detail: { data: event } }), + ); + + // The client should've emitted about the received event + expect((await emittedEvent).getEffectiveEvent()).toEqual(event); + expect(await emittedSync).toEqual(SyncState.Syncing); + // It should've also inserted the event into the room object + const room = client.getRoom("!1:example.org"); + expect(room).not.toBeNull(); + expect(room!.getLiveTimeline().getEvents().map(e => e.getEffectiveEvent())).toEqual([event]); + }); + }); + + describe("messages", () => { + it("requests permissions for specific message types", async () => { + await makeClient({ sendMessage: [MsgType.Text], receiveMessage: [MsgType.Text] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendMessage).toHaveBeenCalledWith(MsgType.Text); + expect(widgetApi.requestCapabilityToReceiveMessage).toHaveBeenCalledWith(MsgType.Text); + }); + + it("requests permissions for all message types", async () => { + await makeClient({ sendMessage: true, receiveMessage: true }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendMessage).toHaveBeenCalledWith(); + expect(widgetApi.requestCapabilityToReceiveMessage).toHaveBeenCalledWith(); + }); + + // No point in testing sending and receiving since it's done exactly the + // same way as non-message events + }); + describe("state events", () => { const event = new MatrixEvent({ type: "org.example.foo", diff --git a/src/client.ts b/src/client.ts index 210075c4220..e3ab3a7dd45 100644 --- a/src/client.ts +++ b/src/client.ts @@ -3980,9 +3980,8 @@ export class MatrixClient extends TypedEventEmitter { + protected encryptAndSendEvent(room: Room, event: MatrixEvent): Promise { let cancelled = false; // Add an extra Promise.resolve() to turn synchronous exceptions into promise rejections, // so that we can handle synchronous and asynchronous exceptions with the @@ -4107,7 +4106,7 @@ export class MatrixClient extends TypedEventEmitter(resolve => this.widgetApi.once("ready", resolve)); @@ -70,9 +110,38 @@ export class RoomWidgetClient extends MatrixClient { super(opts); // Request capabilities for the functionality this client needs to support - if (capabilities.sendState?.length || capabilities.receiveState?.length) { + if ( + capabilities.sendEvent?.length + || capabilities.receiveEvent?.length + || capabilities.sendMessage === true + || (Array.isArray(capabilities.sendMessage) && capabilities.sendMessage.length) + || capabilities.receiveMessage === true + || (Array.isArray(capabilities.receiveMessage) && capabilities.receiveMessage.length) + || capabilities.sendState?.length + || capabilities.receiveState?.length + ) { widgetApi.requestCapabilityForRoomTimeline(roomId); } + capabilities.sendEvent?.forEach(eventType => + widgetApi.requestCapabilityToSendEvent(eventType), + ); + capabilities.receiveEvent?.forEach(eventType => + widgetApi.requestCapabilityToReceiveEvent(eventType), + ); + if (capabilities.sendMessage === true) { + widgetApi.requestCapabilityToSendMessage(); + } else if (Array.isArray(capabilities.sendMessage)) { + capabilities.sendMessage.forEach(msgType => + widgetApi.requestCapabilityToSendMessage(msgType), + ); + } + if (capabilities.receiveMessage === true) { + widgetApi.requestCapabilityToReceiveMessage(); + } else if (Array.isArray(capabilities.receiveMessage)) { + capabilities.receiveMessage.forEach(msgType => + widgetApi.requestCapabilityToReceiveMessage(msgType), + ); + } capabilities.sendState?.forEach(({ eventType, stateKey }) => widgetApi.requestCapabilityToSendState(eventType, stateKey), ); @@ -155,6 +224,19 @@ export class RoomWidgetClient extends MatrixClient { throw new Error(`Unknown room: ${roomIdOrAlias}`); } + protected async encryptAndSendEvent(room: Room, event: MatrixEvent): Promise { + let response: ISendEventFromWidgetResponseData; + try { + response = await this.widgetApi.sendRoomEvent(event.getType(), event.getContent(), room.roomId); + } catch (e) { + this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT); + throw e; + } + + room.updatePendingEvent(event, EventStatus.SENT, response.event_id); + return { event_id: response.event_id }; + } + public async sendStateEvent( roomId: string, eventType: string, From c374ba2367f17a6be6f25cf9ff779281fedfd3a7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 26 Oct 2022 11:45:03 +0100 Subject: [PATCH 278/291] TS strict mode compliance in the call / groupcall code (#2805) * TS strict mode compliance in the call / groupcall code * Also the test * Fix initOpponentCrypto to not panic if it doesn't actually need to init crypto --- spec/unit/webrtc/callEventHandler.spec.ts | 16 +-- spec/unit/webrtc/groupCall.spec.ts | 66 +++++------ src/webrtc/audioContext.ts | 2 +- src/webrtc/call.ts | 127 +++++++++++++--------- src/webrtc/callEventHandler.ts | 22 ++-- src/webrtc/callFeed.ts | 8 +- src/webrtc/groupCall.ts | 79 ++++++++------ src/webrtc/groupCallEventHandler.ts | 8 +- src/webrtc/mediaHandler.ts | 11 +- 9 files changed, 186 insertions(+), 153 deletions(-) diff --git a/spec/unit/webrtc/callEventHandler.spec.ts b/spec/unit/webrtc/callEventHandler.spec.ts index e9bbeea28af..d6012a7a27d 100644 --- a/spec/unit/webrtc/callEventHandler.spec.ts +++ b/spec/unit/webrtc/callEventHandler.spec.ts @@ -99,7 +99,7 @@ describe("CallEventHandler", () => { expect(callEventHandler.callEventBuffer.length).toBe(2); expect(callEventHandler.nextSeqByCall.get("123")).toBe(2); - expect(callEventHandler.toDeviceEventBuffers.get("123").length).toBe(1); + expect(callEventHandler.toDeviceEventBuffers.get("123")?.length).toBe(1); const event4 = new MatrixEvent({ type: EventType.CallCandidates, @@ -112,7 +112,7 @@ describe("CallEventHandler", () => { expect(callEventHandler.callEventBuffer.length).toBe(2); expect(callEventHandler.nextSeqByCall.get("123")).toBe(2); - expect(callEventHandler.toDeviceEventBuffers.get("123").length).toBe(2); + expect(callEventHandler.toDeviceEventBuffers.get("123")?.length).toBe(2); const event5 = new MatrixEvent({ type: EventType.CallCandidates, @@ -125,7 +125,7 @@ describe("CallEventHandler", () => { expect(callEventHandler.callEventBuffer.length).toBe(5); expect(callEventHandler.nextSeqByCall.get("123")).toBe(5); - expect(callEventHandler.toDeviceEventBuffers.get("123").length).toBe(0); + expect(callEventHandler.toDeviceEventBuffers.get("123")?.length).toBe(0); }); it("should ignore a call if invite & hangup come within a single sync", () => { @@ -161,7 +161,7 @@ describe("CallEventHandler", () => { it("should ignore non-call events", async () => { // @ts-ignore Mock handleCallEvent is private jest.spyOn(client.callEventHandler, "handleCallEvent"); - jest.spyOn(client, "checkTurnServers").mockReturnValue(undefined); + jest.spyOn(client, "checkTurnServers").mockReturnValue(Promise.resolve(true)); const room = new Room("!room:id", client, "@user:id"); const timelineData: IRoomTimelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; @@ -186,10 +186,10 @@ describe("CallEventHandler", () => { let room: Room; beforeEach(() => { - room = new Room("!room:id", client, client.getUserId()); + room = new Room("!room:id", client, client.getUserId()!); timelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; - jest.spyOn(client, "checkTurnServers").mockReturnValue(undefined); + jest.spyOn(client, "checkTurnServers").mockReturnValue(Promise.resolve(true)); jest.spyOn(client, "getRoom").mockReturnValue(room); jest.spyOn(room, "getMember").mockReturnValue({ user_id: client.getUserId() } as unknown as RoomMember); @@ -246,10 +246,10 @@ describe("CallEventHandler", () => { await sync(); expect(incomingCallListener).toHaveBeenCalled(); - expect(call.groupCallId).toBe(GROUP_CALL_ID); + expect(call!.groupCallId).toBe(GROUP_CALL_ID); // @ts-ignore Mock opponentDeviceId is private expect(call.opponentDeviceId).toBe(DEVICE_ID); - expect(call.getOpponentSessionId()).toBe(SESSION_ID); + expect(call!.getOpponentSessionId()).toBe(SESSION_ID); // @ts-ignore Mock onIncomingCall is private expect(groupCall.onIncomingCall).toHaveBeenCalledWith(call); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index ae95e4c86e9..75b75c2e490 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -116,8 +116,8 @@ class MockCall { setAudioVideoMuted: jest.fn(), stream: new MockMediaStream("stream"), }; - public remoteUsermediaFeed: CallFeed; - public remoteScreensharingFeed: CallFeed; + public remoteUsermediaFeed?: CallFeed; + public remoteScreensharingFeed?: CallFeed; public reject = jest.fn(); public answerWithCallFeeds = jest.fn(); @@ -128,7 +128,7 @@ class MockCall { on = jest.fn(); removeListener = jest.fn(); - getOpponentMember() { + getOpponentMember(): Partial { return { userId: this.opponentUserId, }; @@ -276,7 +276,7 @@ describe('Group Call', function() { await groupCall.initLocalCallFeed(); - const oldStream = groupCall.localCallFeed.stream as unknown as MockMediaStream; + const oldStream = groupCall.localCallFeed?.stream as unknown as MockMediaStream; // arbitrary values, important part is that they're the same afterwards await groupCall.setLocalVideoMuted(true); @@ -286,7 +286,7 @@ describe('Group Call', function() { groupCall.updateLocalUsermediaStream(newStream); - expect(groupCall.localCallFeed.stream).toBe(newStream); + expect(groupCall.localCallFeed?.stream).toBe(newStream); expect(groupCall.isLocalVideoMuted()).toEqual(true); expect(groupCall.isMicrophoneMuted()).toEqual(false); @@ -474,7 +474,7 @@ describe('Group Call', function() { // we should still be muted at this point because the metadata update hasn't sent expect(groupCall.isMicrophoneMuted()).toEqual(true); expect(mockCall.localUsermediaFeed.setAudioVideoMuted).not.toHaveBeenCalled(); - metadataUpdateResolve(); + metadataUpdateResolve!(); await mutePromise; @@ -500,7 +500,7 @@ describe('Group Call', function() { // we should be muted at this point, before the metadata update has been sent expect(groupCall.isMicrophoneMuted()).toEqual(true); expect(mockCall.localUsermediaFeed.setAudioVideoMuted).toHaveBeenCalled(); - metadataUpdateResolve(); + metadataUpdateResolve!(); await mutePromise; @@ -550,7 +550,7 @@ describe('Group Call', function() { groupCall1.onMemberStateChanged(fakeEvent); groupCall2.onMemberStateChanged(fakeEvent); } - return Promise.resolve(null); + return Promise.resolve({ "event_id": "foo" }); }; client1.sendStateEvent.mockImplementation(fakeSendStateEvents); @@ -644,7 +644,7 @@ describe('Group Call', function() { expect(client1.sendToDevice).toHaveBeenCalled(); const oldCall = groupCall1.getCallByUserId(client2.userId); - oldCall.emit(CallEvent.Hangup, oldCall); + oldCall!.emit(CallEvent.Hangup, oldCall!); client1.sendToDevice.mockClear(); @@ -660,11 +660,11 @@ describe('Group Call', function() { // when we placed the call, we could await on enter which waited for the call to // be made. We don't have that luxury now, so first have to wait for the call // to even be created... - let newCall: MatrixCall; + let newCall: MatrixCall | undefined; while ( (newCall = groupCall1.getCallByUserId(client2.userId)) === undefined || newCall.peerConn === undefined || - newCall.callId == oldCall.callId + newCall.callId == oldCall!.callId ) { await flushPromises(); } @@ -704,7 +704,7 @@ describe('Group Call', function() { groupCall1.setMicrophoneMuted(false); groupCall1.setLocalVideoMuted(false); - const call = groupCall1.getCallByUserId(client2.userId); + const call = groupCall1.getCallByUserId(client2.userId)!; call.isMicrophoneMuted = jest.fn().mockReturnValue(true); call.setMicrophoneMuted = jest.fn(); call.isLocalVideoMuted = jest.fn().mockReturnValue(true); @@ -743,13 +743,13 @@ describe('Group Call', function() { it("should mute local audio when calling setMicrophoneMuted()", async () => { const groupCall = await createAndEnterGroupCall(mockClient, room); - groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); + groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); const setAVMutedArray = groupCall.calls.map(call => { - call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); - return call.localUsermediaFeed.setAudioVideoMuted; + call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); + return call.localUsermediaFeed!.setAudioVideoMuted; }); - const tracksArray = groupCall.calls.reduce((acc, call) => { - acc.push(...call.localUsermediaStream.getAudioTracks()); + const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { + acc.push(...call.localUsermediaStream!.getAudioTracks()); return acc; }, []); const sendMetadataUpdateArray = groupCall.calls.map(call => { @@ -759,8 +759,8 @@ describe('Group Call', function() { await groupCall.setMicrophoneMuted(true); - groupCall.localCallFeed.stream.getAudioTracks().forEach(track => expect(track.enabled).toBe(false)); - expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(true, null); + groupCall.localCallFeed!.stream.getAudioTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(true, null); setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(true, null)); tracksArray.forEach(track => expect(track.enabled).toBe(false)); sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); @@ -771,14 +771,14 @@ describe('Group Call', function() { it("should mute local video when calling setLocalVideoMuted()", async () => { const groupCall = await createAndEnterGroupCall(mockClient, room); - groupCall.localCallFeed.setAudioVideoMuted = jest.fn(); + groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); const setAVMutedArray = groupCall.calls.map(call => { - call.localUsermediaFeed.setAudioVideoMuted = jest.fn(); - call.localUsermediaFeed.isVideoMuted = jest.fn().mockReturnValue(true); - return call.localUsermediaFeed.setAudioVideoMuted; + call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); + call.localUsermediaFeed!.isVideoMuted = jest.fn().mockReturnValue(true); + return call.localUsermediaFeed!.setAudioVideoMuted; }); - const tracksArray = groupCall.calls.reduce((acc, call) => { - acc.push(...call.localUsermediaStream.getVideoTracks()); + const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { + acc.push(...call.localUsermediaStream!.getVideoTracks()); return acc; }, []); const sendMetadataUpdateArray = groupCall.calls.map(call => { @@ -788,8 +788,8 @@ describe('Group Call', function() { await groupCall.setLocalVideoMuted(true); - groupCall.localCallFeed.stream.getVideoTracks().forEach(track => expect(track.enabled).toBe(false)); - expect(groupCall.localCallFeed.setAudioVideoMuted).toHaveBeenCalledWith(null, true); + groupCall.localCallFeed!.stream.getVideoTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(null, true); setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(null, true)); tracksArray.forEach(track => expect(track.enabled).toBe(false)); sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); @@ -827,9 +827,9 @@ describe('Group Call', function() { ])); call.onSDPStreamMetadataChangedReceived(metadataEvent); - const feed = groupCall.getUserMediaFeedByUserId(call.invitee); - expect(feed.isAudioMuted()).toBe(true); - expect(feed.isVideoMuted()).toBe(false); + const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); + expect(feed!.isAudioMuted()).toBe(true); + expect(feed!.isVideoMuted()).toBe(false); groupCall.terminate(); }); @@ -850,9 +850,9 @@ describe('Group Call', function() { ])); call.onSDPStreamMetadataChangedReceived(metadataEvent); - const feed = groupCall.getUserMediaFeedByUserId(call.invitee); - expect(feed.isAudioMuted()).toBe(false); - expect(feed.isVideoMuted()).toBe(true); + const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); + expect(feed!.isAudioMuted()).toBe(false); + expect(feed!.isVideoMuted()).toBe(true); groupCall.terminate(); }); diff --git a/src/webrtc/audioContext.ts b/src/webrtc/audioContext.ts index 10f0dd9490b..8a9ceb15da6 100644 --- a/src/webrtc/audioContext.ts +++ b/src/webrtc/audioContext.ts @@ -38,7 +38,7 @@ export const acquireContext = (): AudioContext => { export const releaseContext = () => { refCount--; if (refCount === 0) { - audioContext.close(); + audioContext?.close(); audioContext = null; } }; diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 961ddcd560f..d3d09ba6f9b 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -331,7 +331,7 @@ function getTransceiverKey(purpose: SDPStreamMetadataPurpose, kind: TransceiverK * @param {MatrixClient} opts.client The Matrix Client instance to send events to. */ export class MatrixCall extends TypedEventEmitter { - public roomId: string; + public roomId?: string; public callId: string; public invitee?: string; public state = CallState.Fledgling; @@ -361,15 +361,15 @@ export class MatrixCall extends TypedEventEmitter(); private inviteOrAnswerSent = false; - private waitForLocalAVStream: boolean; + private waitForLocalAVStream = false; private successor?: MatrixCall; private opponentMember?: RoomMember; private opponentVersion?: number | string; // The party ID of the other side: undefined if we haven't chosen a partner // yet, null if we have but they didn't send a party ID. - private opponentPartyId: string | null; - private opponentCaps: CallCapabilities; - private iceDisconnectedTimeout: ReturnType; + private opponentPartyId: string | null | undefined; + private opponentCaps?: CallCapabilities; + private iceDisconnectedTimeout?: ReturnType; private inviteTimeout?: ReturnType; private readonly removeTrackListeners = new Map void>(); @@ -384,7 +384,7 @@ export class MatrixCall extends TypedEventEmitter; @@ -399,17 +399,21 @@ export class MatrixCall extends TypedEventEmitter; private callLength = 0; - private opponentDeviceId: string; - private opponentDeviceInfo: DeviceInfo; - private opponentSessionId: string; - public groupCallId: string; + private opponentDeviceId?: string; + private opponentDeviceInfo?: DeviceInfo; + private opponentSessionId?: string; + public groupCallId?: string; constructor(opts: CallOpts) { super(); + this.roomId = opts.roomId; this.invitee = opts.invitee; this.client = opts.client; - this.forceTURN = opts.forceTURN; + + if (!this.client.deviceId) throw new Error("Client must have a device ID to start calls"); + + this.forceTURN = opts.forceTURN ?? false; this.ourPartyId = this.client.deviceId; this.opponentDeviceId = opts.opponentDeviceId; this.opponentSessionId = opts.opponentSessionId; @@ -448,7 +452,7 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender)); + const newTransciever = this.peerConn!.getTransceivers().find(t => t.sender === newSender); + if (newTransciever) { + this.transceivers.set(tKey, newTransciever); + } else { + logger.warn("Didn't find a matching transceiver after adding track!"); + } } } } @@ -797,8 +811,8 @@ export class MatrixCall extends TypedEventEmitter { stats.push(item); }); @@ -917,8 +931,8 @@ export class MatrixCall extends TypedEventEmitter track.kind === "video"); const sender = this.transceivers.get(getTransceiverKey( SDPStreamMetadataPurpose.Usermedia, "video", - )).sender; - sender?.replaceTrack(track); + ))?.sender; + sender?.replaceTrack(track ?? null); this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!); this.deleteFeedByStream(this.localScreensharingStream!); @@ -1298,8 +1312,13 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender)); + const newSender = this.peerConn!.addTrack(track, this.localUsermediaStream!); + const newTransciever = this.peerConn!.getTransceivers().find(t => t.sender === newSender); + if (newTransciever) { + this.transceivers.set(tKey, newTransciever); + } else { + logger.warn("Couldn't find matching transceiver for newly added track!"); + } } } } @@ -1436,7 +1455,7 @@ export class MatrixCall extends TypedEventEmitter { const payloadTypeToCodecMap = new Map(); @@ -1570,7 +1589,7 @@ export class MatrixCall extends TypedEventEmitter { - const offer = await this.peerConn.createOffer(); + const offer = await this.peerConn!.createOffer(); this.mungeSdp(offer, getCodecParamMods(this.isPtt)); return offer; } private async createAnswer(): Promise { - const answer = await this.peerConn.createAnswer(); + const answer = await this.peerConn!.createAnswer(); this.mungeSdp(answer, getCodecParamMods(this.isPtt)); return answer; } @@ -1665,7 +1684,7 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug(`Call ${this.callId} ice gathering state changed to ${this.peerConn.iceGatheringState}`); + logger.debug(`Call ${this.callId} ice gathering state changed to ${this.peerConn!.iceGatheringState}`); if (this.peerConn?.iceGatheringState === 'complete') { this.queueCandidate(null); } @@ -1688,10 +1707,12 @@ export class MatrixCall extends TypedEventEmitter void) | null) { this.candidatesEnded = false; this.peerConn!.restartIce(); } else { @@ -2085,7 +2108,7 @@ export class MatrixCall extends TypedEventEmitter { diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 6a3d5339dcc..7c2be58ffc9 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -16,7 +16,7 @@ limitations under the License. import { MatrixEvent } from '../models/event'; import { logger } from '../logger'; -import { CallDirection, CallErrorCode, CallState, createNewMatrixCall, MatrixCall } from './call'; +import { CallDirection, CallError, CallErrorCode, CallState, createNewMatrixCall, MatrixCall } from './call'; import { EventType } from '../@types/event'; import { ClientEvent, MatrixClient } from '../client'; import { MCallAnswer, MCallHangupReject } from "./callEventTypes"; @@ -152,7 +152,7 @@ export class CallEventHandler { this.toDeviceEventBuffers.set(content.call_id, []); } - const buffer = this.toDeviceEventBuffers.get(content.call_id); + const buffer = this.toDeviceEventBuffers.get(content.call_id)!; const index = buffer.findIndex((e) => e.getContent().seq > content.seq); if (index === -1) { @@ -172,7 +172,7 @@ export class CallEventHandler { while (nextEvent && nextEvent.getContent().seq === this.nextSeqByCall.get(callId)) { this.callEventBuffer.push(nextEvent); this.nextSeqByCall.set(callId, nextEvent.getContent().seq + 1); - nextEvent = buffer.shift(); + nextEvent = buffer!.shift(); } } }; @@ -194,7 +194,7 @@ export class CallEventHandler { let opponentDeviceId: string | undefined; - let groupCall: GroupCall; + let groupCall: GroupCall | undefined; if (groupCallId) { groupCall = this.client.groupCallEventHandler.getGroupCallById(groupCallId); @@ -241,7 +241,7 @@ export class CallEventHandler { return; // This invite was meant for another user in the room } - const timeUntilTurnCresExpire = this.client.getTurnServersExpiry() - Date.now(); + const timeUntilTurnCresExpire = (this.client.getTurnServersExpiry() ?? 0) - Date.now(); logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms"); call = createNewMatrixCall( this.client, @@ -267,10 +267,12 @@ export class CallEventHandler { try { await call.initWithInvite(event); } catch (e) { - if (e.code === GroupCallErrorCode.UnknownDevice) { - groupCall?.emit(GroupCallEvent.Error, e); - } else { - logger.error(e); + if (e instanceof CallError) { + if (e.code === GroupCallErrorCode.UnknownDevice) { + groupCall?.emit(GroupCallEvent.Error, e); + } else { + logger.error(e); + } } } this.calls.set(call.callId, call); @@ -292,7 +294,7 @@ export class CallEventHandler { if ( call.roomId === thisCall.roomId && thisCall.direction === CallDirection.Outbound && - call.getOpponentMember().userId === thisCall.invitee && + call.getOpponentMember()?.userId === thisCall.invitee && isCalling ) { existingCall = thisCall; diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 1709102963e..51318c50fed 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -27,7 +27,7 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export interface ICallFeedOpts { client: MatrixClient; - roomId: string; + roomId?: string; userId: string; stream: MediaStream; purpose: SDPStreamMetadataPurpose; @@ -67,7 +67,7 @@ export class CallFeed extends TypedEventEmitter public speakingVolumeSamples: number[]; private client: MatrixClient; - private roomId: string; + private roomId?: string; private audioMuted: boolean; private videoMuted: boolean; private localVolume = 1; @@ -295,8 +295,8 @@ export class CallFeed extends TypedEventEmitter clearTimeout(this.volumeLooperTimeout); this.stream?.removeEventListener("addtrack", this.onAddTrack); if (this.audioContext) { - this.audioContext = null; - this.analyser = null; + this.audioContext = undefined; + this.analyser = undefined; releaseContext(); } this._disposed = true; diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index eb122a395ce..150b71ea340 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -9,6 +9,7 @@ import { CallErrorCode, MatrixCall, setTracksEnabled, createNewMatrixCall, + CallError, } from "./call"; import { RoomMember } from "../models/room-member"; import { Room } from "../models/room"; @@ -56,7 +57,7 @@ export type GroupCallEventHandlerMap = { [GroupCallEvent.UserMediaFeedsChanged]: (feeds: CallFeed[]) => void; [GroupCallEvent.ScreenshareFeedsChanged]: (feeds: CallFeed[]) => void; [GroupCallEvent.LocalScreenshareStateChanged]: ( - isScreensharing: boolean, feed: CallFeed, sourceId: string, + isScreensharing: boolean, feed?: CallFeed, sourceId?: string, ) => void; [GroupCallEvent.LocalMuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; [GroupCallEvent.ParticipantsChanged]: (participants: RoomMember[]) => void; @@ -136,7 +137,7 @@ export enum GroupCallState { interface ICallHandlers { onCallFeedsChanged: (feeds: CallFeed[]) => void; - onCallStateChanged: (state: CallState, oldState: CallState) => void; + onCallStateChanged: (state: CallState, oldState: CallState | undefined) => void; onCallHangup: (call: MatrixCall) => void; onCallReplaced: (newCall: MatrixCall) => void; } @@ -232,7 +233,7 @@ export class GroupCall extends TypedEventEmitter< } public getLocalFeeds(): CallFeed[] { - const feeds = []; + const feeds: CallFeed[] = []; if (this.localCallFeed) feeds.push(this.localCallFeed); if (this.localScreenshareFeed) feeds.push(this.localScreenshareFeed); @@ -311,11 +312,11 @@ export class GroupCall extends TypedEventEmitter< await this.initLocalCallFeed(); } - this.addParticipant(this.room.getMember(this.client.getUserId())); + this.addParticipant(this.room.getMember(this.client.getUserId()!)!); await this.sendMemberStateEvent(); - this.activeSpeaker = null; + this.activeSpeaker = undefined; this.setState(GroupCallState.Entered); @@ -343,7 +344,7 @@ export class GroupCall extends TypedEventEmitter< private dispose() { if (this.localCallFeed) { this.removeUserMediaFeed(this.localCallFeed); - this.localCallFeed = null; + this.localCallFeed = undefined; } if (this.localScreenshareFeed) { @@ -359,7 +360,7 @@ export class GroupCall extends TypedEventEmitter< return; } - this.removeParticipant(this.room.getMember(this.client.getUserId())); + this.removeParticipant(this.room.getMember(this.client.getUserId()!)!); this.removeMemberStateEvent(); @@ -367,7 +368,7 @@ export class GroupCall extends TypedEventEmitter< this.removeCall(this.calls[this.calls.length - 1], CallErrorCode.UserHangup); } - this.activeSpeaker = null; + this.activeSpeaker = undefined; clearTimeout(this.activeSpeakerLoopTimeout); this.retryCallCounts.clear(); @@ -470,7 +471,7 @@ export class GroupCall extends TypedEventEmitter< this.setMicrophoneMuted(true); }, this.pttMaxTransmitTime); } else if (muted && !this.isMicrophoneMuted()) { - clearTimeout(this.transmitTimer); + if (this.transmitTimer !== null) clearTimeout(this.transmitTimer); this.transmitTimer = null; } } @@ -502,7 +503,7 @@ export class GroupCall extends TypedEventEmitter< } for (const call of this.calls) { - setTracksEnabled(call.localUsermediaFeed.stream.getAudioTracks(), !muted); + setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted); } this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); @@ -576,7 +577,7 @@ export class GroupCall extends TypedEventEmitter< this.localScreenshareFeed = new CallFeed({ client: this.client, roomId: this.room.roomId, - userId: this.client.getUserId(), + userId: this.client.getUserId()!, stream, purpose: SDPStreamMetadataPurpose.Screenshare, audioMuted: false, @@ -593,7 +594,7 @@ export class GroupCall extends TypedEventEmitter< // TODO: handle errors await Promise.all(this.calls.map(call => call.pushLocalFeed( - this.localScreenshareFeed.clone(), + this.localScreenshareFeed!.clone(), ))); await this.sendMemberStateEvent(); @@ -603,7 +604,10 @@ export class GroupCall extends TypedEventEmitter< if (opts.throwOnFail) throw error; logger.error("Enabling screensharing error", error); this.emit(GroupCallEvent.Error, - new GroupCallError(GroupCallErrorCode.NoUserMedia, "Failed to get screen-sharing stream: ", error), + new GroupCallError( + GroupCallErrorCode.NoUserMedia, + "Failed to get screen-sharing stream: ", error as Error, + ), ); return false; } @@ -611,8 +615,8 @@ export class GroupCall extends TypedEventEmitter< await Promise.all(this.calls.map(call => { if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); })); - this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); - this.removeScreenshareFeed(this.localScreenshareFeed); + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); + this.removeScreenshareFeed(this.localScreenshareFeed!); this.localScreenshareFeed = undefined; this.localDesktopCapturerSourceId = undefined; await this.sendMemberStateEvent(); @@ -652,8 +656,8 @@ export class GroupCall extends TypedEventEmitter< return; } - const opponentMemberId = newCall.getOpponentMember().userId; - const existingCall = this.getCallByUserId(opponentMemberId); + const opponentMemberId = newCall.getOpponentMember()?.userId; + const existingCall = opponentMemberId ? this.getCallByUserId(opponentMemberId) : null; if (existingCall && existingCall.callId === newCall.callId) { return; @@ -709,7 +713,7 @@ export class GroupCall extends TypedEventEmitter< const res = await send(); // Clear the old interval first, so that it isn't forgot - clearInterval(this.resendMemberStateTimer); + if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); // Resend the state event every so often so it doesn't become stale this.resendMemberStateTimer = setInterval(async () => { logger.log("Resending call member state"); @@ -720,13 +724,13 @@ export class GroupCall extends TypedEventEmitter< } private async removeMemberStateEvent(): Promise { - clearInterval(this.resendMemberStateTimer); + if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); this.resendMemberStateTimer = null; return await this.updateMemberCallState(undefined); } private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { - const localUserId = this.client.getUserId(); + const localUserId = this.client.getUserId()!; const memberState = this.getMemberStateEvents(localUserId)?.getContent(); @@ -766,7 +770,7 @@ export class GroupCall extends TypedEventEmitter< // The member events may be received for another room, which we will ignore. if (event.getRoomId() !== this.room.roomId) return; - const member = this.room.getMember(event.getStateKey()); + const member = this.room.getMember(event.getStateKey()!); if (!member) { logger.warn(`Couldn't find room member for ${event.getStateKey()}: ignoring member state event!`); return; @@ -816,7 +820,7 @@ export class GroupCall extends TypedEventEmitter< }, content["m.expires_ts"] - Date.now())); // Don't process your own member. - const localUserId = this.client.getUserId(); + const localUserId = this.client.getUserId()!; if (member.userId === localUserId) { return; @@ -860,6 +864,11 @@ export class GroupCall extends TypedEventEmitter< }, ); + if (!newCall) { + logger.error("Failed to create call!"); + return; + } + if (existingCall) { logger.debug(`Replacing call ${existingCall.callId} to ${member.userId} with ${newCall.callId}`); this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); @@ -884,7 +893,7 @@ export class GroupCall extends TypedEventEmitter< ); } catch (e) { logger.warn(`Failed to place call to ${member.userId}!`, e); - if (e.code === GroupCallErrorCode.UnknownDevice) { + if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { this.emit(GroupCallEvent.Error, e); } else { this.emit( @@ -904,7 +913,7 @@ export class GroupCall extends TypedEventEmitter< } }; - public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice { + public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice | undefined { const memberStateEvent = this.getMemberStateEvents(userId); if (!memberStateEvent) { @@ -931,7 +940,7 @@ export class GroupCall extends TypedEventEmitter< private onRetryCallLoop = () => { for (const event of this.getMemberStateEvents()) { - const memberId = event.getStateKey(); + const memberId = event.getStateKey()!; const existingCall = this.calls.find((call) => getCallUserId(call) === memberId); const retryCallCount = this.retryCallCounts.get(memberId) || 0; @@ -948,7 +957,7 @@ export class GroupCall extends TypedEventEmitter< * Call Event Handlers */ - public getCallByUserId(userId: string): MatrixCall { + public getCallByUserId(userId: string): MatrixCall | undefined { return this.calls.find((call) => getCallUserId(call) === userId); } @@ -996,7 +1005,7 @@ export class GroupCall extends TypedEventEmitter< const onCallFeedsChanged = () => this.onCallFeedsChanged(call); const onCallStateChanged = - (state: CallState, oldState: CallState) => this.onCallStateChanged(call, state, oldState); + (state: CallState, oldState: CallState | undefined) => this.onCallStateChanged(call, state, oldState); const onCallHangup = this.onCallHangup; const onCallReplaced = (newCall: MatrixCall) => this.replaceCall(call, newCall); @@ -1029,7 +1038,7 @@ export class GroupCall extends TypedEventEmitter< onCallStateChanged, onCallHangup, onCallReplaced, - } = this.callHandlers.get(opponentMemberId); + } = this.callHandlers.get(opponentMemberId)!; call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); call.removeListener(CallEvent.State, onCallStateChanged); @@ -1095,8 +1104,8 @@ export class GroupCall extends TypedEventEmitter< } }; - private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState) => { - const audioMuted = this.localCallFeed.isAudioMuted(); + private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined) => { + const audioMuted = this.localCallFeed!.isAudioMuted(); if ( call.localUsermediaStream && @@ -1105,7 +1114,7 @@ export class GroupCall extends TypedEventEmitter< call.setMicrophoneMuted(audioMuted); } - const videoMuted = this.localCallFeed.isVideoMuted(); + const videoMuted = this.localCallFeed!.isVideoMuted(); if ( call.localUsermediaStream && @@ -1115,7 +1124,7 @@ export class GroupCall extends TypedEventEmitter< } if (state === CallState.Connected) { - this.retryCallCounts.delete(getCallUserId(call)); + this.retryCallCounts.delete(getCallUserId(call)!); } }; @@ -1177,8 +1186,8 @@ export class GroupCall extends TypedEventEmitter< } private onActiveSpeakerLoop = () => { - let topAvg: number; - let nextActiveSpeaker: string; + let topAvg: number | undefined = undefined; + let nextActiveSpeaker: string | undefined = undefined; for (const callFeed of this.userMediaFeeds) { if (callFeed.userId === this.client.getUserId() && this.userMediaFeeds.length > 1) { @@ -1200,7 +1209,7 @@ export class GroupCall extends TypedEventEmitter< } } - if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg > SPEAKING_THRESHOLD) { + if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg && topAvg > SPEAKING_THRESHOLD) { this.activeSpeaker = nextActiveSpeaker; this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); } diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts index e38d7f2b419..86df722895d 100644 --- a/src/webrtc/groupCallEventHandler.ts +++ b/src/webrtc/groupCallEventHandler.ts @@ -91,7 +91,7 @@ export class GroupCallEventHandler { } private getRoomDeferred(roomId: string): RoomDeferred { - let deferred: RoomDeferred = this.roomDeferreds.get(roomId); + let deferred = this.roomDeferreds.get(roomId); if (deferred === undefined) { let resolveFunc: () => void; deferred = { @@ -99,7 +99,7 @@ export class GroupCallEventHandler { resolveFunc = resolve; }), }; - deferred.resolve = resolveFunc; + deferred.resolve = resolveFunc!; this.roomDeferreds.set(roomId, deferred); } @@ -110,7 +110,7 @@ export class GroupCallEventHandler { return this.getRoomDeferred(roomId).prom; } - public getGroupCallById(groupCallId: string): GroupCall { + public getGroupCallById(groupCallId: string): GroupCall | undefined { return [...this.groupCalls.values()].find((groupCall) => groupCall.groupCallId === groupCallId); } @@ -135,7 +135,7 @@ export class GroupCallEventHandler { } logger.info("Group call event handler processed room", room.roomId); - this.getRoomDeferred(room.roomId).resolve(); + this.getRoomDeferred(room.roomId).resolve!(); } private createGroupCallFromRoomStateEvent(event: MatrixEvent): GroupCall | undefined { diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index fa337098f2b..bff6330186f 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -228,8 +228,8 @@ export class MediaHandler extends TypedEventEmitter< this.localUserMediaStream = stream; } } else { - stream = this.localUserMediaStream.clone(); - logger.log(`mediaHandler clone userMediaStream ${this.localUserMediaStream.id} new stream ${ + stream = this.localUserMediaStream!.clone(); + logger.log(`mediaHandler clone userMediaStream ${this.localUserMediaStream?.id} new stream ${ stream.id} shouldRequestAudio ${shouldRequestAudio} shouldRequestVideo ${shouldRequestVideo}`); if (!shouldRequestAudio) { @@ -282,12 +282,11 @@ export class MediaHandler extends TypedEventEmitter< * @param reusable is allowed to be reused by the MediaHandler * @returns {MediaStream} based on passed parameters */ - public async getScreensharingStream(opts: IScreensharingOpts = {}, reusable = true): Promise { + public async getScreensharingStream(opts: IScreensharingOpts = {}, reusable = true): Promise { let stream: MediaStream; if (this.screensharingStreams.length === 0) { const screenshareConstraints = this.getScreenshareContraints(opts); - if (!screenshareConstraints) return null; if (opts.desktopCapturerSourceId) { // We are using Electron @@ -385,7 +384,7 @@ export class MediaHandler extends TypedEventEmitter< if (desktopCapturerSourceId) { logger.debug("Using desktop capturer source", desktopCapturerSourceId); return { - audio, + audio: audio ?? false, video: { mandatory: { chromeMediaSource: "desktop", @@ -396,7 +395,7 @@ export class MediaHandler extends TypedEventEmitter< } else { logger.debug("Not using desktop capturer source"); return { - audio, + audio: audio ?? false, video: true, }; } From 0d211dfbada3f2f3d65273d70e6379fc705f5534 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 26 Oct 2022 11:57:16 +0100 Subject: [PATCH 279/291] Clean up group call tests (#2806) --- spec/unit/webrtc/groupCall.spec.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 75b75c2e490..0c0990aa1f5 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -584,6 +584,8 @@ describe('Group Call', function() { }); afterEach(function() { + groupCall1.leave(); + groupCall2.leave(); jest.useRealTimers(); MockRTCPeerConnection.resetInstances(); From 0f1012278a37d91c04bbda6ffa8c45cf80188977 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 26 Oct 2022 12:01:53 +0100 Subject: [PATCH 280/291] Fix types --- spec/unit/embedded.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts index 3a144215c6a..436909be73c 100644 --- a/spec/unit/embedded.spec.ts +++ b/spec/unit/embedded.spec.ts @@ -28,6 +28,7 @@ import { WidgetApiToWidgetAction, MatrixCapabilities, ITurnServer, + IRoomEvent, } from "matrix-widget-api"; import { createRoomWidgetClient, MsgType } from "../../src/matrix"; @@ -184,7 +185,7 @@ describe("RoomWidgetClient", () => { it("backfills", async () => { widgetApi.readStateEvents.mockImplementation(async (eventType, limit, stateKey) => eventType === "org.example.foo" && (limit ?? Infinity) > 0 && stateKey === "bar" - ? [event] + ? [event as IRoomEvent] : [], ); From 5679c86ca622bbd790d6c6751dd4953e1ab4a853 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 26 Oct 2022 12:33:06 +0100 Subject: [PATCH 281/291] More TS strict mode fixes --- spec/test-utils/webrtc.ts | 18 ++++----- spec/unit/webrtc/call.spec.ts | 38 +++++++++---------- spec/unit/webrtc/callEventHandler.spec.ts | 8 ++-- spec/unit/webrtc/groupCall.spec.ts | 2 +- .../unit/webrtc/groupCallEventHandler.spec.ts | 2 +- src/webrtc/call.ts | 4 +- src/webrtc/callEventHandler.ts | 6 +-- src/webrtc/groupCall.ts | 8 ++-- src/webrtc/mediaHandler.ts | 4 +- tsconfig.json | 1 + 10 files changed, 46 insertions(+), 45 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 6a571b73797..e141e60878a 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -101,12 +101,12 @@ export class MockAudioContext { export class MockRTCPeerConnection { private static instances: MockRTCPeerConnection[] = []; - private negotiationNeededListener: () => void; + private negotiationNeededListener?: () => void; public iceCandidateListener?: (e: RTCPeerConnectionIceEvent) => void; public onTrackListener?: (e: RTCTrackEvent) => void; public needsNegotiation = false; public readyToNegotiate: Promise; - private onReadyToNegotiate: () => void; + private onReadyToNegotiate?: () => void; localDescription: RTCSessionDescription; signalingState: RTCSignalingState = "stable"; public transceivers: MockRTCRtpTransceiver[] = []; @@ -171,7 +171,7 @@ export class MockRTCPeerConnection { getStats() { return []; } addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { this.needsNegotiation = true; - this.onReadyToNegotiate(); + if (this.onReadyToNegotiate) this.onReadyToNegotiate(); const newSender = new MockRTCRtpSender(track); const newReceiver = new MockRTCRtpReceiver(track); @@ -190,7 +190,7 @@ export class MockRTCPeerConnection { removeTrack() { this.needsNegotiation = true; - this.onReadyToNegotiate(); + if (this.onReadyToNegotiate) this.onReadyToNegotiate(); } getTransceivers(): MockRTCRtpTransceiver[] { return this.transceivers; } @@ -217,8 +217,8 @@ export class MockRTCRtpReceiver { export class MockRTCRtpTransceiver { constructor(private peerConn: MockRTCPeerConnection) {} - public sender: RTCRtpSender; - public receiver: RTCRtpReceiver; + public sender?: RTCRtpSender; + public receiver?: RTCRtpReceiver; public set direction(_: string) { this.peerConn.needsNegotiation = true; @@ -234,9 +234,9 @@ export class MockMediaStreamTrack { listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; - public settings: MediaTrackSettings; + public settings?: MediaTrackSettings; - getSettings(): MediaTrackSettings { return this.settings; } + getSettings(): MediaTrackSettings { return this.settings!; } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own // implementation @@ -409,7 +409,7 @@ export class MockCallMatrixClient extends TypedEventEmitter false; checkTurnServers = () => null; - getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); + getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); getRooms = jest.fn().mockReturnValue([]); getRoom = jest.fn(); diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 4d2c50a350a..331830d18ac 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -169,7 +169,7 @@ describe('Call', function() { }, })); - const mockAddIceCandidate = call.peerConn.addIceCandidate = jest.fn(); + const mockAddIceCandidate = call.peerConn!.addIceCandidate = jest.fn(); call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { version: 1, call_id: call.callId, @@ -199,7 +199,7 @@ describe('Call', function() { it('should add candidates received before answer if party ID is correct', async function() { await startVoiceCall(client, call); - const mockAddIceCandidate = call.peerConn.addIceCandidate = jest.fn(); + const mockAddIceCandidate = call.peerConn!.addIceCandidate = jest.fn(); call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { version: 1, @@ -372,12 +372,12 @@ describe('Call', function() { // XXX: Lots of inspecting the prvate state of the call object here const transceivers: Map = (call as any).transceivers; - expect(call.localUsermediaStream.id).toBe("stream"); - expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("new_audio_track"); - expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); + expect(call.localUsermediaStream!.id).toBe("stream"); + expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("new_audio_track"); + expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("video_track"); // call has a function for generating these but we hardcode here to avoid exporting it - expect(transceivers.get("m.usermedia:audio").sender.track.id).toBe("new_audio_track"); - expect(transceivers.get("m.usermedia:video").sender.track.id).toBe("video_track"); + expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("new_audio_track"); + expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("video_track"); }); it("should handle upgrade to video call", async () => { @@ -400,10 +400,10 @@ describe('Call', function() { // XXX: More inspecting private state of the call object const transceivers: Map = (call as any).transceivers; - expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("usermedia_audio_track"); - expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("usermedia_video_track"); - expect(transceivers.get("m.usermedia:audio").sender.track.id).toBe("usermedia_audio_track"); - expect(transceivers.get("m.usermedia:video").sender.track.id).toBe("usermedia_video_track"); + expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("usermedia_audio_track"); + expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("usermedia_video_track"); + expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("usermedia_audio_track"); + expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("usermedia_video_track"); }); it("should handle SDPStreamMetadata changes", async () => { @@ -601,13 +601,13 @@ describe('Call', function() { (call as any).pushRemoteFeed(remoteUsermediaStream); (call as any).pushRemoteFeed(remoteScreensharingStream); - expect(call.localUsermediaFeed.stream).toBe(localUsermediaStream); + expect(call.localUsermediaFeed!.stream).toBe(localUsermediaStream); expect(call.localUsermediaStream).toBe(localUsermediaStream); - expect(call.localScreensharingFeed.stream).toBe(localScreensharingStream); + expect(call.localScreensharingFeed!.stream).toBe(localScreensharingStream); expect(call.localScreensharingStream).toBe(localScreensharingStream); - expect(call.remoteUsermediaFeed.stream).toBe(remoteUsermediaStream); + expect(call.remoteUsermediaFeed!.stream).toBe(remoteUsermediaStream); expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream); - expect(call.remoteScreensharingFeed.stream).toBe(remoteScreensharingStream); + expect(call.remoteScreensharingFeed!.stream).toBe(remoteScreensharingStream); expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream); expect(call.hasRemoteUserMediaAudioTrack).toBe(false); }); @@ -1214,8 +1214,8 @@ describe('Call', function() { MockRTCPeerConnection.triggerAllNegotiations(); - const mockVideoSender = call.peerConn.getSenders().find(s => s.track.kind === "video"); - const mockReplaceTrack = mockVideoSender.replaceTrack = jest.fn(); + const mockVideoSender = call.peerConn!.getSenders().find(s => s.track!.kind === "video"); + const mockReplaceTrack = mockVideoSender!.replaceTrack = jest.fn(); await call.setScreensharingEnabled(true); @@ -1336,7 +1336,7 @@ describe('Call', function() { await call.placeVoiceCall(); - (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener( + (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!( { streams: [], track: new MockMediaStreamTrack("track_ev", "audio") } as unknown as RTCTrackEvent, ); @@ -1359,7 +1359,7 @@ describe('Call', function() { })); const stream = new MockMediaStream("stream_ev", [new MockMediaStreamTrack("track_ev", "audio")]); - (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener( + (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!( { streams: [stream], track: stream.getAudioTracks()[0] } as unknown as RTCTrackEvent, ); diff --git a/spec/unit/webrtc/callEventHandler.spec.ts b/spec/unit/webrtc/callEventHandler.spec.ts index d6012a7a27d..b7687cdc2fc 100644 --- a/spec/unit/webrtc/callEventHandler.spec.ts +++ b/spec/unit/webrtc/callEventHandler.spec.ts @@ -50,8 +50,8 @@ describe("CallEventHandler", () => { }); afterEach(() => { - client.callEventHandler.stop(); - client.groupCallEventHandler.stop(); + client.callEventHandler!.stop(); + client.groupCallEventHandler!.stop(); }); const sync = async () => { @@ -63,7 +63,7 @@ describe("CallEventHandler", () => { }; it("should enforce inbound toDevice message ordering", async () => { - const callEventHandler = client.callEventHandler; + const callEventHandler = client.callEventHandler!; const event1 = new MatrixEvent({ type: EventType.CallInvite, content: { @@ -227,7 +227,7 @@ describe("CallEventHandler", () => { const DEVICE_ID = "device_id"; incomingCallListener.mockImplementation((c) => call = c); - jest.spyOn(client.groupCallEventHandler, "getGroupCallById").mockReturnValue(groupCall); + jest.spyOn(client.groupCallEventHandler!, "getGroupCallById").mockReturnValue(groupCall); // @ts-ignore Mock onIncomingCall is private jest.spyOn(groupCall, "onIncomingCall"); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 0c0990aa1f5..e3c850c7040 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -939,7 +939,7 @@ describe('Group Call', function() { groupCall.leave(); const call = new MockCall(room.roomId, groupCall.groupCallId); - mockClient.callEventHandler.calls = new Map([ + mockClient.callEventHandler!.calls = new Map([ [call.callId, call.typed()], ]); await groupCall.enter(); diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts index e948f8afaaa..6712b0f09f0 100644 --- a/spec/unit/webrtc/groupCallEventHandler.spec.ts +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -74,7 +74,7 @@ describe('Group Call Event Handler', function() { { roomId: FAKE_ROOM_ID } as unknown as RoomState, ); - const groupCall = groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID); + const groupCall = groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID)!; expect(groupCall.state).toBe(GroupCallState.LocalCallFeedUninitialized); diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 3e9950a4345..6e43b48be02 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -2462,7 +2462,7 @@ export class MatrixCall extends TypedEventEmitter { diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 7c2be58ffc9..93035e62499 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -183,11 +183,11 @@ export class CallEventHandler { const content = event.getContent(); const callRoomId = ( event.getRoomId() || - this.client.groupCallEventHandler.getGroupCallById(content.conf_id)?.room?.roomId + this.client.groupCallEventHandler!.getGroupCallById(content.conf_id)?.room?.roomId ); const groupCallId = content.conf_id; const type = event.getType() as EventType; - const senderId = event.getSender(); + const senderId = event.getSender()!; const weSentTheEvent = senderId === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; //console.info("RECV %s content=%s", type, JSON.stringify(content)); @@ -196,7 +196,7 @@ export class CallEventHandler { let groupCall: GroupCall | undefined; if (groupCallId) { - groupCall = this.client.groupCallEventHandler.getGroupCallById(groupCallId); + groupCall = this.client.groupCallEventHandler!.getGroupCallById(groupCallId); if (!groupCall) { logger.warn(`Cannot find a group call ${groupCallId} for event ${type}. Ignoring event.`); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 150b71ea340..c2fd3c4fb1b 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -207,7 +207,7 @@ export class GroupCall extends TypedEventEmitter< } public async create() { - this.client.groupCallEventHandler.groupCalls.set(this.room.roomId, this); + this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); await this.client.sendStateEvent( this.room.roomId, @@ -324,7 +324,7 @@ export class GroupCall extends TypedEventEmitter< this.client.on(CallEventHandlerEvent.Incoming, this.onIncomingCall); - const calls = this.client.callEventHandler.calls.values(); + const calls = this.client.callEventHandler!.calls.values(); for (const call of calls) { this.onIncomingCall(call); @@ -406,7 +406,7 @@ export class GroupCall extends TypedEventEmitter< } this.participants = []; - this.client.groupCallEventHandler.groupCalls.delete(this.room.roomId); + this.client.groupCallEventHandler!.groupCalls.delete(this.room.roomId); if (emitStateEvent) { const existingStateEvent = this.room.currentState.getStateEvents( @@ -699,7 +699,7 @@ export class GroupCall extends TypedEventEmitter< "m.call_id": this.groupCallId, "m.devices": [ { - "device_id": this.client.getDeviceId(), + "device_id": this.client.getDeviceId()!, "session_id": this.client.getSessionId(), "feeds": this.getLocalFeeds().map((feed) => ({ purpose: feed.purpose, diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index a07016bf61e..3cea257caca 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -123,7 +123,7 @@ export class MediaHandler extends TypedEventEmitter< this.userMediaStreams = []; this.localUserMediaStream = undefined; - for (const call of this.client.callEventHandler.calls.values()) { + for (const call of this.client.callEventHandler!.calls.values()) { if (call.callHasEnded() || !callMediaStreamParams.has(call.callId)) { continue; } @@ -140,7 +140,7 @@ export class MediaHandler extends TypedEventEmitter< await call.updateLocalUsermediaStream(stream); } - for (const groupCall of this.client.groupCallEventHandler.groupCalls.values()) { + for (const groupCall of this.client.groupCallEventHandler!.groupCalls.values()) { if (!groupCall.localCallFeed) { continue; } diff --git a/tsconfig.json b/tsconfig.json index 69c3f0196b5..9776cfbab2e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,6 +8,7 @@ "noImplicitAny": false, "noUnusedLocals": true, "noEmit": true, + "strict": true, "declaration": true }, "include": [ From 87115d181dded34c38526e632ff522696d6aefbb Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 26 Oct 2022 12:34:18 +0100 Subject: [PATCH 282/291] Don't commit the strict mode flag --- tsconfig.json | 1 - 1 file changed, 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 9776cfbab2e..69c3f0196b5 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -8,7 +8,6 @@ "noImplicitAny": false, "noUnusedLocals": true, "noEmit": true, - "strict": true, "declaration": true }, "include": [ From dbdaa1540a9b2e78e45eb6f27347aaf41d67b059 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 26 Oct 2022 17:50:20 -0400 Subject: [PATCH 283/291] Let leave requests outlive the window --- spec/unit/webrtc/groupCall.spec.ts | 20 ++++++++++++++++++++ src/client.ts | 4 +++- src/http-api/fetch.ts | 4 +++- src/http-api/interface.ts | 1 + src/webrtc/groupCall.ts | 11 ++++++++--- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index e3c850c7040..692fbdbfb4c 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -212,12 +212,32 @@ describe('Group Call', function() { ], }), FAKE_USER_ID_1, + false, ); } finally { groupCall.leave(); } }); + it("sends member state event to room on leave", async () => { + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + + await groupCall.create(); + await groupCall.enter(); + mockSendState.mockClear(); + + groupCall.leave(); + expect(mockSendState).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.GroupCallMemberPrefix, + expect.objectContaining({ "m.calls": [] }), + FAKE_USER_ID_1, + true, // Request should outlive the window + ); + }); + it("starts with mic unmuted in regular calls", async () => { try { await groupCall.create(); diff --git a/src/client.ts b/src/client.ts index 022fee66359..4b814b2f4f2 100644 --- a/src/client.ts +++ b/src/client.ts @@ -7517,6 +7517,7 @@ export class MatrixClient extends TypedEventEmitter { const pathParams = { $roomId: roomId, @@ -7535,7 +7537,7 @@ export class MatrixClient extends TypedEventEmitter { method: Method, url: URL | string, body?: Body, - opts: Pick = {}, + opts: Pick = {}, ): Promise> { const headers = Object.assign({}, opts.headers || {}); const json = opts.json ?? true; @@ -252,6 +252,7 @@ export class FetchHttpApi { } const timeout = opts.localTimeoutMs ?? this.opts.localTimeoutMs; + const keepAlive = opts.keepAlive ?? false; const signals = [ this.abortController.signal, ]; @@ -284,6 +285,7 @@ export class FetchHttpApi { referrerPolicy: "no-referrer", cache: "no-cache", credentials: "omit", // we send credentials via headers + keepalive: keepAlive, }); } catch (e) { if ((e).name === "AbortError") { diff --git a/src/http-api/interface.ts b/src/http-api/interface.ts index c798bec0d6c..37217947027 100644 --- a/src/http-api/interface.ts +++ b/src/http-api/interface.ts @@ -38,6 +38,7 @@ export interface IRequestOpts { headers?: Record; abortSignal?: AbortSignal; localTimeoutMs?: number; + keepAlive?: boolean; // defaults to false json?: boolean; // defaults to true // Set to true to prevent the request function from emitting a Session.logged_out event. diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index c2fd3c4fb1b..0715062288a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -726,10 +726,13 @@ export class GroupCall extends TypedEventEmitter< private async removeMemberStateEvent(): Promise { if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); this.resendMemberStateTimer = null; - return await this.updateMemberCallState(undefined); + return await this.updateMemberCallState(undefined, true); } - private async updateMemberCallState(memberCallState?: IGroupCallRoomMemberCallState): Promise { + private async updateMemberCallState( + memberCallState?: IGroupCallRoomMemberCallState, + keepAlive = false, + ): Promise { const localUserId = this.client.getUserId()!; const memberState = this.getMemberStateEvents(localUserId)?.getContent(); @@ -758,7 +761,9 @@ export class GroupCall extends TypedEventEmitter< "m.expires_ts": Date.now() + CALL_MEMBER_STATE_TIMEOUT, }; - return this.client.sendStateEvent(this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId); + return this.client.sendStateEvent( + this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId, keepAlive, + ); } public onMemberStateChanged = async (event: MatrixEvent) => { From d979302e9bc341015cc4d3be8608d140229b2fd7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 27 Oct 2022 09:37:41 +0100 Subject: [PATCH 284/291] A few more strict mode fixes --- src/client.ts | 8 ++++---- src/embedded.ts | 18 +++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/client.ts b/src/client.ts index 022fee66359..a35083c44c5 100644 --- a/src/client.ts +++ b/src/client.ts @@ -1607,7 +1607,7 @@ export class MatrixClient extends TypedEventEmitter { - return this.groupCallEventHandler.waitUntilRoomReadyForGroupCalls(roomId); + return this.groupCallEventHandler!.waitUntilRoomReadyForGroupCalls(roomId); } /** @@ -1616,7 +1616,7 @@ export class MatrixClient extends TypedEventEmitter { if (this.isInitialSyncComplete()) { - this.callEventHandler.start(); - this.groupCallEventHandler.start(); + this.callEventHandler!.start(); + this.groupCallEventHandler!.start(); this.off(ClientEvent.Sync, this.startCallEventHandler); } }; diff --git a/src/embedded.ts b/src/embedded.ts index 256ccd2bd5e..d1dfb50bf2f 100644 --- a/src/embedded.ts +++ b/src/embedded.ts @@ -96,9 +96,9 @@ export interface ICapabilities { * @experimental This class is considered unstable! */ export class RoomWidgetClient extends MatrixClient { - private room: Room; + private room?: Room; private widgetApiReady = new Promise(resolve => this.widgetApi.once("ready", resolve)); - private lifecycle: AbortController; + private lifecycle?: AbortController; private syncState: SyncState | null = null; constructor( @@ -197,7 +197,7 @@ export class RoomWidgetClient extends MatrixClient { const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey, [this.roomId]); const events = rawEvents.map(rawEvent => new MatrixEvent(rawEvent as Partial)); - await this.syncApi.injectRoomEvents(this.room, [], events); + await this.syncApi!.injectRoomEvents(this.room!, [], events); events.forEach(event => { this.emit(ClientEvent.Event, event); logger.info(`Backfilled event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); @@ -216,11 +216,11 @@ export class RoomWidgetClient extends MatrixClient { this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); super.stopClient(); - this.lifecycle.abort(); // Signal to other async tasks that the client has stopped + this.lifecycle!.abort(); // Signal to other async tasks that the client has stopped } public async joinRoom(roomIdOrAlias: string): Promise { - if (roomIdOrAlias === this.roomId) return this.room; + if (roomIdOrAlias === this.roomId) return this.room!; throw new Error(`Unknown room: ${roomIdOrAlias}`); } @@ -284,7 +284,7 @@ export class RoomWidgetClient extends MatrixClient { } // Overridden since we 'sync' manually without the sync API - public getSyncState(): SyncState { + public getSyncState(): SyncState | null { return this.syncState; } @@ -305,7 +305,7 @@ export class RoomWidgetClient extends MatrixClient { // send us events from other rooms if this widget is always on screen if (ev.detail.data.room_id === this.roomId) { const event = new MatrixEvent(ev.detail.data as Partial); - await this.syncApi.injectRoomEvents(this.room, [], [event]); + await this.syncApi!.injectRoomEvents(this.room!, [], [event]); this.emit(ClientEvent.Event, event); this.setSyncState(SyncState.Syncing); logger.info(`Received event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); @@ -336,7 +336,7 @@ export class RoomWidgetClient extends MatrixClient { private async watchTurnServers() { const servers = this.widgetApi.getTurnServers(); const onClientStopped = () => servers.return(undefined); - this.lifecycle.signal.addEventListener("abort", onClientStopped); + this.lifecycle!.signal.addEventListener("abort", onClientStopped); try { for await (const server of servers) { @@ -351,7 +351,7 @@ export class RoomWidgetClient extends MatrixClient { } catch (e) { logger.warn("Error watching TURN servers", e); } finally { - this.lifecycle.signal.removeEventListener("abort", onClientStopped); + this.lifecycle!.signal.removeEventListener("abort", onClientStopped); } } } From 4c6e1e5c21b8a2995b2d1cd67e04a65b43455ea7 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Thu, 27 Oct 2022 08:19:41 -0400 Subject: [PATCH 285/291] Replace the keepAlive flag with request options --- spec/unit/webrtc/groupCall.spec.ts | 4 ++-- src/client.ts | 7 ++++--- src/webrtc/groupCall.ts | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index 692fbdbfb4c..e9a9c8a93ea 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -212,7 +212,7 @@ describe('Group Call', function() { ], }), FAKE_USER_ID_1, - false, + { keepAlive: false }, ); } finally { groupCall.leave(); @@ -234,7 +234,7 @@ describe('Group Call', function() { EventType.GroupCallMemberPrefix, expect.objectContaining({ "m.calls": [] }), FAKE_USER_ID_1, - true, // Request should outlive the window + { keepAlive: true }, // Request should outlive the window ); }); diff --git a/src/client.ts b/src/client.ts index 4b814b2f4f2..60c4d55f41a 100644 --- a/src/client.ts +++ b/src/client.ts @@ -65,6 +65,7 @@ import { FileType, UploadResponse, HTTPError, + IRequestOpts, } from "./http-api"; import { Crypto, @@ -7517,7 +7518,7 @@ export class MatrixClient extends TypedEventEmitter { const pathParams = { $roomId: roomId, @@ -7537,7 +7538,7 @@ export class MatrixClient extends TypedEventEmitter Date: Mon, 31 Oct 2022 12:08:19 -0400 Subject: [PATCH 286/291] Resolve races between initLocalCallFeed and leave Unfortunately there are still other methods that could race with leave and result in broken group call state, such as enter and terminate. For the future, should consider writing a more careful specification of how the whole group call state machine is meant to work. --- spec/unit/webrtc/groupCall.spec.ts | 9 ++++++++- src/webrtc/groupCall.ts | 13 +++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index e9a9c8a93ea..f83de56e5c2 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -23,7 +23,7 @@ import { Room, RoomMember, } from '../../../src'; -import { GroupCall, GroupCallEvent } from "../../../src/webrtc/groupCall"; +import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall"; import { MatrixClient } from "../../../src/client"; import { installWebRTCMocks, @@ -174,6 +174,13 @@ describe('Group Call', function() { groupCall.leave(); }); + it("stops initializing local call feed when leaving", async () => { + const initPromise = groupCall.initLocalCallFeed(); + groupCall.leave(); + await expect(initPromise).rejects.toBeDefined(); + expect(groupCall.state).toBe(GroupCallState.LocalCallFeedUninitialized); + }); + it("sends state event to room when creating", async () => { await groupCall.create(); diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index 5b3bb1ee056..cb73b7183e3 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -257,13 +257,26 @@ export class GroupCall extends TypedEventEmitter< let stream: MediaStream; + let disposed = false; + const onState = (state: GroupCallState) => { + if (state === GroupCallState.LocalCallFeedUninitialized) { + disposed = true; + } + }; + this.on(GroupCallEvent.GroupCallStateChanged, onState); + try { stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); } catch (error) { this.setState(GroupCallState.LocalCallFeedUninitialized); throw error; + } finally { + this.off(GroupCallEvent.GroupCallStateChanged, onState); } + // The call could've been disposed while we were waiting + if (disposed) throw new Error("Group call disposed"); + const userId = this.client.getUserId()!; const callFeed = new CallFeed({ From cb8123dec7c30e3f156e9b1fcaf05ffc6ba8d655 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 1 Nov 2022 16:07:07 +0000 Subject: [PATCH 287/291] Add public/private modifiers --- src/webrtc/callEventHandler.ts | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index 93035e62499..13fe4fd5c7e 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -36,13 +36,14 @@ export type CallEventHandlerEventHandlerMap = { }; export class CallEventHandler { - client: MatrixClient; - calls: Map; - callEventBuffer: MatrixEvent[]; - candidateEventsByCall: Map>; - nextSeqByCall: Map = new Map(); - toDeviceEventBuffers: Map> = new Map(); - + // XXX: Most of these are only public because of the tests + public calls: Map; + public callEventBuffer: MatrixEvent[]; + public nextSeqByCall: Map = new Map(); + public toDeviceEventBuffers: Map> = new Map(); + + private client: MatrixClient; + private candidateEventsByCall: Map>; private eventBufferPromiseChain?: Promise; constructor(client: MatrixClient) { From 476f6f78b16e5c597c9d1c0ceca5caf4f4fe296d Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 1 Nov 2022 16:14:48 +0000 Subject: [PATCH 288/291] Add more access modifiers --- spec/test-utils/webrtc.ts | 154 +++++++++++++++-------------- spec/unit/webrtc/groupCall.spec.ts | 8 +- src/webrtc/groupCall.ts | 2 +- 3 files changed, 83 insertions(+), 81 deletions(-) diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index e141e60878a..66110701139 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -84,18 +84,18 @@ export const USERMEDIA_STREAM_ID = "mock_stream_from_media_handler"; export const SCREENSHARE_STREAM_ID = "mock_screen_stream_from_media_handler"; class MockMediaStreamAudioSourceNode { - connect() {} + public connect() {} } class MockAnalyser { - getFloatFrequencyData() { return 0.0; } + public getFloatFrequencyData() { return 0.0; } } export class MockAudioContext { constructor() {} - createAnalyser() { return new MockAnalyser(); } - createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } - close() {} + public createAnalyser() { return new MockAnalyser(); } + public createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } + public close() {} } export class MockRTCPeerConnection { @@ -107,8 +107,8 @@ export class MockRTCPeerConnection { public needsNegotiation = false; public readyToNegotiate: Promise; private onReadyToNegotiate?: () => void; - localDescription: RTCSessionDescription; - signalingState: RTCSignalingState = "stable"; + public localDescription: RTCSessionDescription; + public signalingState: RTCSignalingState = "stable"; public transceivers: MockRTCRtpTransceiver[] = []; public static triggerAllNegotiations(): void { @@ -139,7 +139,7 @@ export class MockRTCPeerConnection { MockRTCPeerConnection.instances.push(this); } - addEventListener(type: string, listener: () => void) { + public addEventListener(type: string, listener: () => void) { if (type === 'negotiationneeded') { this.negotiationNeededListener = listener; } else if (type == 'icecandidate') { @@ -148,28 +148,28 @@ export class MockRTCPeerConnection { this.onTrackListener = listener; } } - createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } - createOffer() { + public createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } + public createOffer() { return Promise.resolve({ type: 'offer', sdp: DUMMY_SDP, }); } - createAnswer() { + public createAnswer() { return Promise.resolve({ type: 'answer', sdp: DUMMY_SDP, }); } - setRemoteDescription() { + public setRemoteDescription() { return Promise.resolve(); } - setLocalDescription() { + public setLocalDescription() { return Promise.resolve(); } - close() { } - getStats() { return []; } - addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { + public close() { } + public getStats() { return []; } + public addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { this.needsNegotiation = true; if (this.onReadyToNegotiate) this.onReadyToNegotiate(); @@ -184,19 +184,21 @@ export class MockRTCPeerConnection { return newTransceiver; } - addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { + public addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { return this.addTransceiver(track).sender as unknown as MockRTCRtpSender; } - removeTrack() { + public removeTrack() { this.needsNegotiation = true; if (this.onReadyToNegotiate) this.onReadyToNegotiate(); } - getTransceivers(): MockRTCRtpTransceiver[] { return this.transceivers; } - getSenders(): MockRTCRtpSender[] { return this.transceivers.map(t => t.sender as unknown as MockRTCRtpSender); } + public getTransceivers(): MockRTCRtpTransceiver[] { return this.transceivers; } + public getSenders(): MockRTCRtpSender[] { + return this.transceivers.map(t => t.sender as unknown as MockRTCRtpSender); + } - doNegotiation() { + public doNegotiation() { if (this.needsNegotiation && this.negotiationNeededListener) { this.needsNegotiation = false; this.negotiationNeededListener(); @@ -207,7 +209,7 @@ export class MockRTCPeerConnection { export class MockRTCRtpSender { constructor(public track: MockMediaStreamTrack) { } - replaceTrack(track: MockMediaStreamTrack) { this.track = track; } + public replaceTrack(track: MockMediaStreamTrack) { this.track = track; } } export class MockRTCRtpReceiver { @@ -224,38 +226,38 @@ export class MockRTCRtpTransceiver { this.peerConn.needsNegotiation = true; } - setCodecPreferences = jest.fn(); + public setCodecPreferences = jest.fn(); } export class MockMediaStreamTrack { constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { } - stop = jest.fn(); + public stop = jest.fn(); - listeners: [string, (...args: any[]) => any][] = []; + public listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; public settings?: MediaTrackSettings; - getSettings(): MediaTrackSettings { return this.settings!; } + public getSettings(): MediaTrackSettings { return this.settings!; } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own // implementation - dispatchEvent(eventType: string) { + public dispatchEvent(eventType: string) { this.listeners.forEach(([t, c]) => { if (t !== eventType) return; c(); }); } - addEventListener(eventType: string, callback: (...args: any[]) => any) { + public addEventListener(eventType: string, callback: (...args: any[]) => any) { this.listeners.push([eventType, callback]); } - removeEventListener(eventType: string, callback: (...args: any[]) => any) { + public removeEventListener(eventType: string, callback: (...args: any[]) => any) { this.listeners.filter(([t, c]) => { return t !== eventType || c !== callback; }); } - typed(): MediaStreamTrack { return this as unknown as MediaStreamTrack; } + public typed(): MediaStreamTrack { return this as unknown as MediaStreamTrack; } } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own @@ -266,42 +268,42 @@ export class MockMediaStream { private tracks: MockMediaStreamTrack[] = [], ) {} - listeners: [string, (...args: any[]) => any][] = []; + public listeners: [string, (...args: any[]) => any][] = []; public isStopped = false; - dispatchEvent(eventType: string) { + public dispatchEvent(eventType: string) { this.listeners.forEach(([t, c]) => { if (t !== eventType) return; c(); }); } - getTracks() { return this.tracks; } - getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); } - getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); } - addEventListener(eventType: string, callback: (...args: any[]) => any) { + public getTracks() { return this.tracks; } + public getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); } + public getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); } + public addEventListener(eventType: string, callback: (...args: any[]) => any) { this.listeners.push([eventType, callback]); } - removeEventListener(eventType: string, callback: (...args: any[]) => any) { + public removeEventListener(eventType: string, callback: (...args: any[]) => any) { this.listeners.filter(([t, c]) => { return t !== eventType || c !== callback; }); } - addTrack(track: MockMediaStreamTrack) { + public addTrack(track: MockMediaStreamTrack) { this.tracks.push(track); this.dispatchEvent("addtrack"); } - removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } + public removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } - clone(): MediaStream { + public clone(): MediaStream { return new MockMediaStream(this.id + ".clone", this.tracks).typed(); } - isCloneOf(stream: MediaStream) { + public isCloneOf(stream: MediaStream) { return this.id === stream.id + ".clone"; } // syntactic sugar for typing - typed(): MediaStream { + public typed(): MediaStream { return this as unknown as MediaStream; } } @@ -311,14 +313,14 @@ export class MockMediaDeviceInfo { public kind: "audioinput" | "videoinput" | "audiooutput", ) { } - typed(): MediaDeviceInfo { return this as unknown as MediaDeviceInfo; } + public typed(): MediaDeviceInfo { return this as unknown as MediaDeviceInfo; } } export class MockMediaHandler { public userMediaStreams: MockMediaStream[] = []; public screensharingStreams: MockMediaStream[] = []; - getUserMediaStream(audio: boolean, video: boolean) { + public getUserMediaStream(audio: boolean, video: boolean) { const tracks: MockMediaStreamTrack[] = []; if (audio) tracks.push(new MockMediaStreamTrack("usermedia_audio_track", "audio")); if (video) tracks.push(new MockMediaStreamTrack("usermedia_video_track", "video")); @@ -327,10 +329,10 @@ export class MockMediaHandler { this.userMediaStreams.push(stream); return stream; } - stopUserMediaStream(stream: MockMediaStream) { + public stopUserMediaStream(stream: MockMediaStream) { stream.isStopped = true; } - getScreensharingStream = jest.fn((opts?: IScreensharingOpts) => { + public getScreensharingStream = jest.fn((opts?: IScreensharingOpts) => { const tracks = [new MockMediaStreamTrack("screenshare_video_track", "video")]; if (opts?.audio) tracks.push(new MockMediaStreamTrack("screenshare_audio_track", "audio")); @@ -338,31 +340,31 @@ export class MockMediaHandler { this.screensharingStreams.push(stream); return stream; }); - stopScreensharingStream(stream: MockMediaStream) { + public stopScreensharingStream(stream: MockMediaStream) { stream.isStopped = true; } - hasAudioDevice() { return true; } - hasVideoDevice() { return true; } - stopAllStreams() {} + public hasAudioDevice() { return true; } + public hasVideoDevice() { return true; } + public stopAllStreams() {} - typed(): MediaHandler { return this as unknown as MediaHandler; } + public typed(): MediaHandler { return this as unknown as MediaHandler; } } export class MockMediaDevices { - enumerateDevices = jest.fn, []>().mockResolvedValue([ + public enumerateDevices = jest.fn, []>().mockResolvedValue([ new MockMediaDeviceInfo("audioinput").typed(), new MockMediaDeviceInfo("videoinput").typed(), ]); - getUserMedia = jest.fn, [MediaStreamConstraints]>().mockReturnValue( + public getUserMedia = jest.fn, [MediaStreamConstraints]>().mockReturnValue( Promise.resolve(new MockMediaStream("local_stream").typed()), ); - getDisplayMedia = jest.fn, [DisplayMediaStreamConstraints]>().mockReturnValue( + public getDisplayMedia = jest.fn, [DisplayMediaStreamConstraints]>().mockReturnValue( Promise.resolve(new MockMediaStream("local_display_stream").typed()), ); - typed(): MediaDevices { return this as unknown as MediaDevices; } + public typed(): MediaDevices { return this as unknown as MediaDevices; } } type EmittedEvents = CallEventHandlerEvent | CallEvent | ClientEvent | RoomStateEvent | GroupCallEventHandlerEvent; @@ -379,44 +381,44 @@ export class MockCallMatrixClient extends TypedEventEmitter(), }; - callEventHandler = { + public callEventHandler = { calls: new Map(), }; - sendStateEvent = jest.fn, [ + public sendStateEvent = jest.fn, [ roomId: string, eventType: EventType, content: any, statekey: string, ]>(); - sendToDevice = jest.fn, [ + public sendToDevice = jest.fn, [ eventType: string, contentMap: { [userId: string]: { [deviceId: string]: Record } }, txnId?: string, ]>(); - getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } + public getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } - getUserId(): string { return this.userId; } + public getUserId(): string { return this.userId; } - getDeviceId(): string { return this.deviceId; } - getSessionId(): string { return this.sessionId; } + public getDeviceId(): string { return this.deviceId; } + public getSessionId(): string { return this.sessionId; } - getTurnServers = () => []; - isFallbackICEServerAllowed = () => false; - reEmitter = new ReEmitter(new TypedEventEmitter()); - getUseE2eForGroupCall = () => false; - checkTurnServers = () => null; + public getTurnServers = () => []; + public isFallbackICEServerAllowed = () => false; + public reEmitter = new ReEmitter(new TypedEventEmitter()); + public getUseE2eForGroupCall = () => false; + public checkTurnServers = () => null; - getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); + public getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); - getRooms = jest.fn().mockReturnValue([]); - getRoom = jest.fn(); + public getRooms = jest.fn().mockReturnValue([]); + public getRoom = jest.fn(); - typed(): MatrixClient { return this as unknown as MatrixClient; } + public typed(): MatrixClient { return this as unknown as MatrixClient; } - emitRoomState(event: MatrixEvent, state: RoomState): void { + public emitRoomState(event: MatrixEvent, state: RoomState): void { this.emit( RoomStateEvent.Events, event, @@ -432,10 +434,10 @@ export class MockCallFeed { public stream: MockMediaStream, ) {} - measureVolumeActivity(val: boolean) {} - dispose() {} + public measureVolumeActivity(val: boolean) {} + public dispose() {} - typed(): CallFeed { + public typed(): CallFeed { return this as unknown as CallFeed; } } diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts index f83de56e5c2..fa84490c146 100644 --- a/spec/unit/webrtc/groupCall.spec.ts +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -125,16 +125,16 @@ class MockCall { public sendMetadataUpdate = jest.fn(); - on = jest.fn(); - removeListener = jest.fn(); + public on = jest.fn(); + public removeListener = jest.fn(); - getOpponentMember(): Partial { + public getOpponentMember(): Partial { return { userId: this.opponentUserId, }; } - typed(): MatrixCall { return this as unknown as MatrixCall; } + public typed(): MatrixCall { return this as unknown as MatrixCall; } } describe('Group Call', function() { diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index cb73b7183e3..fa26eab840a 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -71,7 +71,7 @@ export enum GroupCallErrorCode { } export class GroupCallError extends Error { - code: string; + public code: string; constructor(code: GroupCallErrorCode, msg: string, err?: Error) { // Still don't think there's any way to have proper nested errors From 7ec726e10be835588d4b188fcd3d137b4690d79a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 1 Nov 2022 16:18:14 +0000 Subject: [PATCH 289/291] Give everything that isn't web rtc back to element-web --- .github/CODEOWNERS | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index dd7aeb18c10..f28b852a788 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,4 @@ -* @matrix-org/element-call-reviewers +* @matrix-org/element-web + +/src/webrtc @matrix-org/element-call-reviewers +/spec/*/webrtc @matrix-org/element-call-reviewers From 32d535c2b11298c9938f4a533b60ba9b6064c97f Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Wed, 2 Nov 2022 22:43:35 -0400 Subject: [PATCH 290/291] Don't remove our own member for a split second when entering a call --- src/webrtc/groupCall.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts index fa26eab840a..e20d388a2bc 100644 --- a/src/webrtc/groupCall.ts +++ b/src/webrtc/groupCall.ts @@ -794,6 +794,10 @@ export class GroupCall extends TypedEventEmitter< return; } + // Don't process your own member. + const localUserId = this.client.getUserId()!; + if (member.userId === localUserId) return; + logger.debug(`Processing member state event for ${member.userId}`); const ignore = () => { @@ -837,13 +841,6 @@ export class GroupCall extends TypedEventEmitter< this.removeParticipant(member); }, content["m.expires_ts"] - Date.now())); - // Don't process your own member. - const localUserId = this.client.getUserId()!; - - if (member.userId === localUserId) { - return; - } - // Only initiate a call with a user who has a userId that is lexicographically // less than your own. Otherwise, that user will call you. if (member.userId < localUserId) { From 2dda837db62d413fbb9f04b6703bccc719a0c9d0 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 4 Nov 2022 14:44:21 +0000 Subject: [PATCH 291/291] Fix strict mode errors --- src/crypto/index.ts | 2 +- src/sliding-sync-sdk.ts | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/crypto/index.ts b/src/crypto/index.ts index 34f108a39c2..e718d2eb097 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -128,7 +128,7 @@ export interface IBootstrapCrossSigningOpts { } export interface ICryptoCallbacks { - getCrossSigningKey?: (keyType: string, pubKey: string) => Promise; + getCrossSigningKey?: (keyType: string, pubKey: string) => Promise; saveCrossSigningKeys?: (keys: Record) => void; shouldUpgradeDeviceVerifications?: ( users: Record diff --git a/src/sliding-sync-sdk.ts b/src/sliding-sync-sdk.ts index 06efc0a8418..962d824a8e5 100644 --- a/src/sliding-sync-sdk.ts +++ b/src/sliding-sync-sdk.ts @@ -396,7 +396,7 @@ export class SlidingSyncSdk { public createRoom(roomId: string): Room { // XXX cargoculted from sync.ts const { timelineSupport } = this.client; - const room = new Room(roomId, this.client, this.client.getUserId(), { + const room = new Room(roomId, this.client, this.client.getUserId()!, { lazyLoadMembers: this.opts.lazyLoadMembers, pendingEventOrdering: this.opts.pendingEventOrdering, timelineSupport, @@ -428,7 +428,7 @@ export class SlidingSyncSdk { RoomStateEvent.Update, ]); room.currentState.on(RoomStateEvent.NewMember, (event, state, member) => { - member.user = this.client.getUser(member.userId); + member.user = this.client.getUser(member.userId) ?? undefined; this.client.reEmitter.reEmit(member, [ RoomMemberEvent.Name, RoomMemberEvent.Typing,