diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 2c068fff330..f28b852a788 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1,4 @@ * @matrix-org/element-web + +/src/webrtc @matrix-org/element-call-reviewers +/spec/*/webrtc @matrix-org/element-call-reviewers diff --git a/package.json b/package.json index 3fcea052a5a..ac83abf6f8c 100644 --- a/package.json +++ b/package.json @@ -55,13 +55,16 @@ ], "dependencies": { "@babel/runtime": "^7.12.5", + "@types/sdp-transform": "^2.4.5", "another-json": "^0.2.0", "bs58": "^5.0.0", "content-type": "^1.0.4", "loglevel": "^1.7.1", "matrix-events-sdk": "0.0.1-beta.7", + "matrix-widget-api": "^1.0.0", "p-retry": "4", "qs": "^6.9.6", + "sdp-transform": "^2.14.1", "unhomoglyph": "^1.0.6" }, "devDependencies": { @@ -102,6 +105,7 @@ "exorcist": "^2.0.0", "fake-indexeddb": "^4.0.0", "jest": "^29.0.0", + "jest-environment-jsdom": "^28.1.3", "jest-localstorage-mock": "^2.4.6", "jest-mock": "^29.0.0", "jsdoc": "^3.6.6", diff --git a/spec/test-utils/flushPromises.ts b/spec/test-utils/flushPromises.ts new file mode 100644 index 00000000000..8512410aa9f --- /dev/null +++ b/spec/test-utils/flushPromises.ts @@ -0,0 +1,28 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Jest now uses @sinonjs/fake-timers which exposes tickAsync() and a number of +// other async methods which break the event loop, letting scheduled promise +// callbacks run. Unfortunately, Jest doesn't expose these, so we have to do +// it manually (this is what sinon does under the hood). We do both in a loop +// until the thing we expect happens: hopefully this is the least flakey way +// and avoids assuming anything about the app's behaviour. +const realSetTimeout = setTimeout; +export function flushPromises() { + return new Promise(r => { + realSetTimeout(r, 1); + }); +} diff --git a/spec/test-utils/webrtc.ts b/spec/test-utils/webrtc.ts index 0244e870c4b..66110701139 100644 --- a/spec/test-utils/webrtc.ts +++ b/spec/test-utils/webrtc.ts @@ -14,6 +14,32 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { + ClientEvent, + ClientEventHandlerMap, + EventType, + GroupCall, + GroupCallIntent, + GroupCallType, + IContent, + ISendEventResponse, + MatrixClient, + MatrixEvent, + Room, + RoomState, + RoomStateEvent, + RoomStateEventHandlerMap, +} from "../../src"; +import { TypedEventEmitter } from "../../src/models/typed-event-emitter"; +import { ReEmitter } from "../../src/ReEmitter"; +import { SyncState } from "../../src/sync"; +import { CallEvent, CallEventHandlerMap, MatrixCall } from "../../src/webrtc/call"; +import { CallEventHandlerEvent, CallEventHandlerEventHandlerMap } from "../../src/webrtc/callEventHandler"; +import { CallFeed } from "../../src/webrtc/callFeed"; +import { GroupCallEventHandlerMap } from "../../src/webrtc/groupCall"; +import { GroupCallEventHandlerEvent } from "../../src/webrtc/groupCallEventHandler"; +import { IScreensharingOpts, MediaHandler } from "../../src/webrtc/mediaHandler"; + export const DUMMY_SDP = ( "v=0\r\n" + "o=- 5022425983810148698 2 IN IP4 127.0.0.1\r\n" + @@ -54,8 +80,50 @@ export const DUMMY_SDP = ( "a=ssrc:3619738545 cname:2RWtmqhXLdoF4sOi\r\n" ); +export const USERMEDIA_STREAM_ID = "mock_stream_from_media_handler"; +export const SCREENSHARE_STREAM_ID = "mock_screen_stream_from_media_handler"; + +class MockMediaStreamAudioSourceNode { + public connect() {} +} + +class MockAnalyser { + public getFloatFrequencyData() { return 0.0; } +} + +export class MockAudioContext { + constructor() {} + public createAnalyser() { return new MockAnalyser(); } + public createMediaStreamSource() { return new MockMediaStreamAudioSourceNode(); } + public close() {} +} + export class MockRTCPeerConnection { - localDescription: RTCSessionDescription; + private static instances: MockRTCPeerConnection[] = []; + + private negotiationNeededListener?: () => void; + public iceCandidateListener?: (e: RTCPeerConnectionIceEvent) => void; + public onTrackListener?: (e: RTCTrackEvent) => void; + public needsNegotiation = false; + public readyToNegotiate: Promise; + private onReadyToNegotiate?: () => void; + public localDescription: RTCSessionDescription; + public signalingState: RTCSignalingState = "stable"; + public transceivers: MockRTCRtpTransceiver[] = []; + + public static triggerAllNegotiations(): void { + for (const inst of this.instances) { + inst.doNegotiation(); + } + } + + public static hasAnyPendingNegotiations(): boolean { + return this.instances.some(i => i.needsNegotiation); + } + + public static resetInstances() { + this.instances = []; + } constructor() { this.localDescription = { @@ -63,34 +131,133 @@ export class MockRTCPeerConnection { type: 'offer', toJSON: function() { }, }; + + this.readyToNegotiate = new Promise(resolve => { + this.onReadyToNegotiate = resolve; + }); + + MockRTCPeerConnection.instances.push(this); } - addEventListener() { } - createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } - createOffer() { - return Promise.resolve({}); + public addEventListener(type: string, listener: () => void) { + if (type === 'negotiationneeded') { + this.negotiationNeededListener = listener; + } else if (type == 'icecandidate') { + this.iceCandidateListener = listener; + } else if (type == 'track') { + this.onTrackListener = listener; + } + } + public createDataChannel(label: string, opts: RTCDataChannelInit) { return { label, ...opts }; } + public createOffer() { + return Promise.resolve({ + type: 'offer', + sdp: DUMMY_SDP, + }); + } + public createAnswer() { + return Promise.resolve({ + type: 'answer', + sdp: DUMMY_SDP, + }); } - setRemoteDescription() { + public setRemoteDescription() { return Promise.resolve(); } - setLocalDescription() { + public setLocalDescription() { return Promise.resolve(); } - close() { } - getStats() { return []; } - addTrack(track: MockMediaStreamTrack) { return new MockRTCRtpSender(track); } + public close() { } + public getStats() { return []; } + public addTransceiver(track: MockMediaStreamTrack): MockRTCRtpTransceiver { + this.needsNegotiation = true; + if (this.onReadyToNegotiate) this.onReadyToNegotiate(); + + const newSender = new MockRTCRtpSender(track); + const newReceiver = new MockRTCRtpReceiver(track); + + const newTransceiver = new MockRTCRtpTransceiver(this); + newTransceiver.sender = newSender as unknown as RTCRtpSender; + newTransceiver.receiver = newReceiver as unknown as RTCRtpReceiver; + + this.transceivers.push(newTransceiver); + + return newTransceiver; + } + public addTrack(track: MockMediaStreamTrack): MockRTCRtpSender { + return this.addTransceiver(track).sender as unknown as MockRTCRtpSender; + } + + public removeTrack() { + this.needsNegotiation = true; + if (this.onReadyToNegotiate) this.onReadyToNegotiate(); + } + + public getTransceivers(): MockRTCRtpTransceiver[] { return this.transceivers; } + public getSenders(): MockRTCRtpSender[] { + return this.transceivers.map(t => t.sender as unknown as MockRTCRtpSender); + } + + public doNegotiation() { + if (this.needsNegotiation && this.negotiationNeededListener) { + this.needsNegotiation = false; + this.negotiationNeededListener(); + } + } } export class MockRTCRtpSender { constructor(public track: MockMediaStreamTrack) { } - replaceTrack(track: MockMediaStreamTrack) { this.track = track; } + public replaceTrack(track: MockMediaStreamTrack) { this.track = track; } +} + +export class MockRTCRtpReceiver { + constructor(public track: MockMediaStreamTrack) { } +} + +export class MockRTCRtpTransceiver { + constructor(private peerConn: MockRTCPeerConnection) {} + + public sender?: RTCRtpSender; + public receiver?: RTCRtpReceiver; + + public set direction(_: string) { + this.peerConn.needsNegotiation = true; + } + + public setCodecPreferences = jest.fn(); } export class MockMediaStreamTrack { constructor(public readonly id: string, public readonly kind: "audio" | "video", public enabled = true) { } - stop() { } + public stop = jest.fn(); + + public listeners: [string, (...args: any[]) => any][] = []; + public isStopped = false; + public settings?: MediaTrackSettings; + + public getSettings(): MediaTrackSettings { return this.settings!; } + + // XXX: Using EventTarget in jest doesn't seem to work, so we write our own + // implementation + public dispatchEvent(eventType: string) { + this.listeners.forEach(([t, c]) => { + if (t !== eventType) return; + c(); + }); + } + public addEventListener(eventType: string, callback: (...args: any[]) => any) { + this.listeners.push([eventType, callback]); + } + public removeEventListener(eventType: string, callback: (...args: any[]) => any) { + this.listeners.filter(([t, c]) => { + return t !== eventType || c !== callback; + }); + } + + public typed(): MediaStreamTrack { return this as unknown as MediaStreamTrack; } } // XXX: Using EventTarget in jest doesn't seem to work, so we write our own @@ -101,46 +268,236 @@ export class MockMediaStream { private tracks: MockMediaStreamTrack[] = [], ) {} - listeners: [string, (...args: any[]) => any][] = []; + public listeners: [string, (...args: any[]) => any][] = []; + public isStopped = false; - dispatchEvent(eventType: string) { + public dispatchEvent(eventType: string) { this.listeners.forEach(([t, c]) => { if (t !== eventType) return; c(); }); } - getTracks() { return this.tracks; } - getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); } - getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); } - addEventListener(eventType: string, callback: (...args: any[]) => any) { + public getTracks() { return this.tracks; } + public getAudioTracks() { return this.tracks.filter((track) => track.kind === "audio"); } + public getVideoTracks() { return this.tracks.filter((track) => track.kind === "video"); } + public addEventListener(eventType: string, callback: (...args: any[]) => any) { this.listeners.push([eventType, callback]); } - removeEventListener(eventType: string, callback: (...args: any[]) => any) { + public removeEventListener(eventType: string, callback: (...args: any[]) => any) { this.listeners.filter(([t, c]) => { return t !== eventType || c !== callback; }); } - addTrack(track: MockMediaStreamTrack) { + public addTrack(track: MockMediaStreamTrack) { this.tracks.push(track); this.dispatchEvent("addtrack"); } - removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } + public removeTrack(track: MockMediaStreamTrack) { this.tracks.splice(this.tracks.indexOf(track), 1); } + + public clone(): MediaStream { + return new MockMediaStream(this.id + ".clone", this.tracks).typed(); + } + + public isCloneOf(stream: MediaStream) { + return this.id === stream.id + ".clone"; + } + + // syntactic sugar for typing + public typed(): MediaStream { + return this as unknown as MediaStream; + } } export class MockMediaDeviceInfo { constructor( - public kind: "audio" | "video", + public kind: "audioinput" | "videoinput" | "audiooutput", ) { } + + public typed(): MediaDeviceInfo { return this as unknown as MediaDeviceInfo; } } export class MockMediaHandler { - getUserMediaStream(audio: boolean, video: boolean) { + public userMediaStreams: MockMediaStream[] = []; + public screensharingStreams: MockMediaStream[] = []; + + public getUserMediaStream(audio: boolean, video: boolean) { const tracks: MockMediaStreamTrack[] = []; - if (audio) tracks.push(new MockMediaStreamTrack("audio_track", "audio")); - if (video) tracks.push(new MockMediaStreamTrack("video_track", "video")); + if (audio) tracks.push(new MockMediaStreamTrack("usermedia_audio_track", "audio")); + if (video) tracks.push(new MockMediaStreamTrack("usermedia_video_track", "video")); + + const stream = new MockMediaStream(USERMEDIA_STREAM_ID, tracks); + this.userMediaStreams.push(stream); + return stream; + } + public stopUserMediaStream(stream: MockMediaStream) { + stream.isStopped = true; + } + public getScreensharingStream = jest.fn((opts?: IScreensharingOpts) => { + const tracks = [new MockMediaStreamTrack("screenshare_video_track", "video")]; + if (opts?.audio) tracks.push(new MockMediaStreamTrack("screenshare_audio_track", "audio")); + + const stream = new MockMediaStream(SCREENSHARE_STREAM_ID, tracks); + this.screensharingStreams.push(stream); + return stream; + }); + public stopScreensharingStream(stream: MockMediaStream) { + stream.isStopped = true; + } + public hasAudioDevice() { return true; } + public hasVideoDevice() { return true; } + public stopAllStreams() {} + + public typed(): MediaHandler { return this as unknown as MediaHandler; } +} + +export class MockMediaDevices { + public enumerateDevices = jest.fn, []>().mockResolvedValue([ + new MockMediaDeviceInfo("audioinput").typed(), + new MockMediaDeviceInfo("videoinput").typed(), + ]); + + public getUserMedia = jest.fn, [MediaStreamConstraints]>().mockReturnValue( + Promise.resolve(new MockMediaStream("local_stream").typed()), + ); + + public getDisplayMedia = jest.fn, [DisplayMediaStreamConstraints]>().mockReturnValue( + Promise.resolve(new MockMediaStream("local_display_stream").typed()), + ); + + public typed(): MediaDevices { return this as unknown as MediaDevices; } +} + +type EmittedEvents = CallEventHandlerEvent | CallEvent | ClientEvent | RoomStateEvent | GroupCallEventHandlerEvent; +type EmittedEventMap = CallEventHandlerEventHandlerMap & + CallEventHandlerMap & + ClientEventHandlerMap & + RoomStateEventHandlerMap & + GroupCallEventHandlerMap; + +export class MockCallMatrixClient extends TypedEventEmitter { + public mediaHandler = new MockMediaHandler(); + + constructor(public userId: string, public deviceId: string, public sessionId: string) { + super(); + } + + public groupCallEventHandler = { + groupCalls: new Map(), + }; + + public callEventHandler = { + calls: new Map(), + }; + + public sendStateEvent = jest.fn, [ + roomId: string, eventType: EventType, content: any, statekey: string, + ]>(); + public sendToDevice = jest.fn, [ + eventType: string, + contentMap: { [userId: string]: { [deviceId: string]: Record } }, + txnId?: string, + ]>(); + + public getMediaHandler(): MediaHandler { return this.mediaHandler.typed(); } + + public getUserId(): string { return this.userId; } + + public getDeviceId(): string { return this.deviceId; } + public getSessionId(): string { return this.sessionId; } + + public getTurnServers = () => []; + public isFallbackICEServerAllowed = () => false; + public reEmitter = new ReEmitter(new TypedEventEmitter()); + public getUseE2eForGroupCall = () => false; + public checkTurnServers = () => null; + + public getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); + + public getRooms = jest.fn().mockReturnValue([]); + public getRoom = jest.fn(); + + public typed(): MatrixClient { return this as unknown as MatrixClient; } + + public emitRoomState(event: MatrixEvent, state: RoomState): void { + this.emit( + RoomStateEvent.Events, + event, + state, + null, + ); + } +} - return new MockMediaStream("mock_stream_from_media_handler", tracks); +export class MockCallFeed { + constructor( + public userId: string, + public stream: MockMediaStream, + ) {} + + public measureVolumeActivity(val: boolean) {} + public dispose() {} + + public typed(): CallFeed { + return this as unknown as CallFeed; } - stopUserMediaStream() { } - hasAudioDevice() { return true; } +} + +export function installWebRTCMocks() { + global.navigator = { + mediaDevices: new MockMediaDevices().typed(), + } as unknown as Navigator; + + global.window = { + // @ts-ignore Mock + RTCPeerConnection: MockRTCPeerConnection, + // @ts-ignore Mock + RTCSessionDescription: {}, + // @ts-ignore Mock + RTCIceCandidate: {}, + getUserMedia: () => new MockMediaStream("local_stream"), + }; + // @ts-ignore Mock + global.document = {}; + + // @ts-ignore Mock + global.AudioContext = MockAudioContext; + + // @ts-ignore Mock + global.RTCRtpReceiver = { + getCapabilities: jest.fn().mockReturnValue({ + codecs: [], + headerExtensions: [], + }), + }; + + // @ts-ignore Mock + global.RTCRtpSender = { + getCapabilities: jest.fn().mockReturnValue({ + codecs: [], + headerExtensions: [], + }), + }; +} + +export function makeMockGroupCallStateEvent(roomId: string, groupCallId: string, content: IContent = { + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, +}): MatrixEvent { + return { + getType: jest.fn().mockReturnValue(EventType.GroupCallPrefix), + getRoomId: jest.fn().mockReturnValue(roomId), + getTs: jest.fn().mockReturnValue(0), + getContent: jest.fn().mockReturnValue(content), + getStateKey: jest.fn().mockReturnValue(groupCallId), + } as unknown as MatrixEvent; +} + +export function makeMockGroupCallMemberStateEvent(roomId: string, groupCallId: string): MatrixEvent { + return { + getType: jest.fn().mockReturnValue(EventType.GroupCallMemberPrefix), + getRoomId: jest.fn().mockReturnValue(roomId), + getTs: jest.fn().mockReturnValue(0), + getContent: jest.fn().mockReturnValue({}), + getStateKey: jest.fn().mockReturnValue(groupCallId), + } as unknown as MatrixEvent; } diff --git a/spec/unit/embedded.spec.ts b/spec/unit/embedded.spec.ts new file mode 100644 index 00000000000..436909be73c --- /dev/null +++ b/spec/unit/embedded.spec.ts @@ -0,0 +1,331 @@ +/** + * @jest-environment jsdom + */ + +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// We have to use EventEmitter here to mock part of the matrix-widget-api +// project, which doesn't know about our TypeEventEmitter implementation at all +// eslint-disable-next-line no-restricted-imports +import { EventEmitter } from "events"; +import { MockedObject } from "jest-mock"; +import { + WidgetApi, + WidgetApiToWidgetAction, + MatrixCapabilities, + ITurnServer, + IRoomEvent, +} from "matrix-widget-api"; + +import { createRoomWidgetClient, MsgType } from "../../src/matrix"; +import { MatrixClient, ClientEvent, ITurnServer as IClientTurnServer } from "../../src/client"; +import { SyncState } from "../../src/sync"; +import { ICapabilities } from "../../src/embedded"; +import { MatrixEvent } from "../../src/models/event"; +import { ToDeviceBatch } from "../../src/models/ToDeviceMessage"; +import { DeviceInfo } from "../../src/crypto/deviceinfo"; + +class MockWidgetApi extends EventEmitter { + public start = jest.fn(); + public requestCapability = jest.fn(); + public requestCapabilities = jest.fn(); + public requestCapabilityForRoomTimeline = jest.fn(); + public requestCapabilityToSendEvent = jest.fn(); + public requestCapabilityToReceiveEvent = jest.fn(); + public requestCapabilityToSendMessage = jest.fn(); + public requestCapabilityToReceiveMessage = jest.fn(); + public requestCapabilityToSendState = jest.fn(); + public requestCapabilityToReceiveState = jest.fn(); + public requestCapabilityToSendToDevice = jest.fn(); + public requestCapabilityToReceiveToDevice = jest.fn(); + public sendRoomEvent = jest.fn(() => ({ event_id: `$${Math.random()}` })); + public sendStateEvent = jest.fn(); + public sendToDevice = jest.fn(); + public readStateEvents = jest.fn(() => []); + public getTurnServers = jest.fn(() => []); + + public transport = { reply: jest.fn() }; +} + +describe("RoomWidgetClient", () => { + let widgetApi: MockedObject; + let client: MatrixClient; + + beforeEach(() => { + widgetApi = new MockWidgetApi() as unknown as MockedObject; + }); + + afterEach(() => { + client.stopClient(); + }); + + const makeClient = async (capabilities: ICapabilities): Promise => { + const baseUrl = "https://example.org"; + client = createRoomWidgetClient(widgetApi, capabilities, "!1:example.org", { baseUrl }); + expect(widgetApi.start).toHaveBeenCalled(); // needs to have been called early in order to not miss messages + widgetApi.emit("ready"); + await client.startClient(); + }; + + describe("events", () => { + it("sends", async () => { + await makeClient({ sendEvent: ["org.matrix.rageshake_request"] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendEvent).toHaveBeenCalledWith("org.matrix.rageshake_request"); + await client.sendEvent("!1:example.org", "org.matrix.rageshake_request", { request_id: 123 }); + expect(widgetApi.sendRoomEvent).toHaveBeenCalledWith( + "org.matrix.rageshake_request", { request_id: 123 }, "!1:example.org", + ); + }); + + it("receives", async () => { + const event = new MatrixEvent({ + type: "org.matrix.rageshake_request", + event_id: "$pduhfiidph", + room_id: "!1:example.org", + sender: "@alice:example.org", + content: { request_id: 123 }, + }).getEffectiveEvent(); + + await makeClient({ receiveEvent: ["org.matrix.rageshake_request"] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToReceiveEvent).toHaveBeenCalledWith("org.matrix.rageshake_request"); + + const emittedEvent = new Promise(resolve => client.once(ClientEvent.Event, resolve)); + const emittedSync = new Promise(resolve => client.once(ClientEvent.Sync, resolve)); + widgetApi.emit( + `action:${WidgetApiToWidgetAction.SendEvent}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.SendEvent}`, { detail: { data: event } }), + ); + + // The client should've emitted about the received event + expect((await emittedEvent).getEffectiveEvent()).toEqual(event); + expect(await emittedSync).toEqual(SyncState.Syncing); + // It should've also inserted the event into the room object + const room = client.getRoom("!1:example.org"); + expect(room).not.toBeNull(); + expect(room!.getLiveTimeline().getEvents().map(e => e.getEffectiveEvent())).toEqual([event]); + }); + }); + + describe("messages", () => { + it("requests permissions for specific message types", async () => { + await makeClient({ sendMessage: [MsgType.Text], receiveMessage: [MsgType.Text] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendMessage).toHaveBeenCalledWith(MsgType.Text); + expect(widgetApi.requestCapabilityToReceiveMessage).toHaveBeenCalledWith(MsgType.Text); + }); + + it("requests permissions for all message types", async () => { + await makeClient({ sendMessage: true, receiveMessage: true }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendMessage).toHaveBeenCalledWith(); + expect(widgetApi.requestCapabilityToReceiveMessage).toHaveBeenCalledWith(); + }); + + // No point in testing sending and receiving since it's done exactly the + // same way as non-message events + }); + + describe("state events", () => { + const event = new MatrixEvent({ + type: "org.example.foo", + event_id: "$sfkjfsksdkfsd", + room_id: "!1:example.org", + sender: "@alice:example.org", + state_key: "bar", + content: { hello: "world" }, + }).getEffectiveEvent(); + + it("sends", async () => { + await makeClient({ sendState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToSendState).toHaveBeenCalledWith("org.example.foo", "bar"); + await client.sendStateEvent("!1:example.org", "org.example.foo", { hello: "world" }, "bar"); + expect(widgetApi.sendStateEvent).toHaveBeenCalledWith( + "org.example.foo", "bar", { hello: "world" }, "!1:example.org", + ); + }); + + it("receives", async () => { + await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); + + const emittedEvent = new Promise(resolve => client.once(ClientEvent.Event, resolve)); + const emittedSync = new Promise(resolve => client.once(ClientEvent.Sync, resolve)); + widgetApi.emit( + `action:${WidgetApiToWidgetAction.SendEvent}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.SendEvent}`, { detail: { data: event } }), + ); + + // The client should've emitted about the received event + expect((await emittedEvent).getEffectiveEvent()).toEqual(event); + expect(await emittedSync).toEqual(SyncState.Syncing); + // It should've also inserted the event into the room object + const room = client.getRoom("!1:example.org"); + expect(room).not.toBeNull(); + expect(room!.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + }); + + it("backfills", async () => { + widgetApi.readStateEvents.mockImplementation(async (eventType, limit, stateKey) => + eventType === "org.example.foo" && (limit ?? Infinity) > 0 && stateKey === "bar" + ? [event as IRoomEvent] + : [], + ); + + await makeClient({ receiveState: [{ eventType: "org.example.foo", stateKey: "bar" }] }); + expect(widgetApi.requestCapabilityForRoomTimeline).toHaveBeenCalledWith("!1:example.org"); + expect(widgetApi.requestCapabilityToReceiveState).toHaveBeenCalledWith("org.example.foo", "bar"); + + const room = client.getRoom("!1:example.org"); + expect(room).not.toBeNull(); + expect(room!.currentState.getStateEvents("org.example.foo", "bar").getEffectiveEvent()).toEqual(event); + }); + }); + + describe("to-device messages", () => { + const unencryptedContentMap = { + "@alice:example.org": { "*": { hello: "alice!" } }, + "@bob:example.org": { bobDesktop: { hello: "bob!" } }, + }; + + it("sends unencrypted (sendToDevice)", async () => { + await makeClient({ sendToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo"); + + await client.sendToDevice("org.example.foo", unencryptedContentMap); + expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", false, unencryptedContentMap); + }); + + it("sends unencrypted (queueToDevice)", async () => { + await makeClient({ sendToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo"); + + const batch: ToDeviceBatch = { + eventType: "org.example.foo", + batch: [ + { userId: "@alice:example.org", deviceId: "*", payload: { hello: "alice!" } }, + { userId: "@bob:example.org", deviceId: "bobDesktop", payload: { hello: "bob!" } }, + ], + }; + await client.queueToDevice(batch); + expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", false, unencryptedContentMap); + }); + + it("sends encrypted (encryptAndSendToDevices)", async () => { + await makeClient({ sendToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToSendToDevice).toHaveBeenCalledWith("org.example.foo"); + + const payload = { type: "org.example.foo", hello: "world" }; + await client.encryptAndSendToDevices( + [ + { userId: "@alice:example.org", deviceInfo: new DeviceInfo("aliceWeb") }, + { userId: "@bob:example.org", deviceInfo: new DeviceInfo("bobDesktop") }, + ], + payload, + ); + expect(widgetApi.sendToDevice).toHaveBeenCalledWith("org.example.foo", true, { + "@alice:example.org": { aliceWeb: payload }, + "@bob:example.org": { bobDesktop: payload }, + }); + }); + + it.each([ + { encrypted: false, title: "unencrypted" }, + { encrypted: true, title: "encrypted" }, + ])("receives $title", async ({ encrypted }) => { + await makeClient({ receiveToDevice: ["org.example.foo"] }); + expect(widgetApi.requestCapabilityToReceiveToDevice).toHaveBeenCalledWith("org.example.foo"); + + const event = { + type: "org.example.foo", + sender: "@alice:example.org", + encrypted, + content: { hello: "world" }, + }; + + const emittedEvent = new Promise(resolve => client.once(ClientEvent.ToDeviceEvent, resolve)); + const emittedSync = new Promise(resolve => client.once(ClientEvent.Sync, resolve)); + widgetApi.emit( + `action:${WidgetApiToWidgetAction.SendToDevice}`, + new CustomEvent(`action:${WidgetApiToWidgetAction.SendToDevice}`, { detail: { data: event } }), + ); + + expect((await emittedEvent).getEffectiveEvent()).toEqual({ + type: event.type, + sender: event.sender, + content: event.content, + }); + expect((await emittedEvent).isEncrypted()).toEqual(encrypted); + expect(await emittedSync).toEqual(SyncState.Syncing); + }); + }); + + it("gets TURN servers", async () => { + const server1: ITurnServer = { + uris: [ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp", + ], + username: "1443779631:@user:example.com", + password: "JlKfBy1QwLrO20385QyAtEyIv0=", + }; + const server2: ITurnServer = { + uris: [ + "turn:turn.example.com:3478?transport=udp", + "turn:10.20.30.40:3478?transport=tcp", + "turns:10.20.30.40:443?transport=tcp", + ], + username: "1448999322:@user:example.com", + password: "hunter2", + }; + const clientServer1: IClientTurnServer = { + urls: server1.uris, + username: server1.username, + credential: server1.password, + }; + const clientServer2: IClientTurnServer = { + urls: server2.uris, + username: server2.username, + credential: server2.password, + }; + + let emitServer2: () => void; + const getServer2 = new Promise(resolve => emitServer2 = () => resolve(server2)); + widgetApi.getTurnServers.mockImplementation(async function* () { + yield server1; + yield await getServer2; + }); + + await makeClient({ turnServers: true }); + expect(widgetApi.requestCapability).toHaveBeenCalledWith(MatrixCapabilities.MSC3846TurnServers); + + // The first server should've arrived immediately + expect(client.getTurnServers()).toEqual([clientServer1]); + + // Subsequent servers arrive asynchronously and should emit an event + const emittedServer = new Promise(resolve => + client.once(ClientEvent.TurnServers, resolve), + ); + emitServer2!(); + expect(await emittedServer).toEqual([clientServer2]); + expect(client.getTurnServers()).toEqual([clientServer2]); + }); +}); diff --git a/spec/unit/matrix-client.spec.ts b/spec/unit/matrix-client.spec.ts index 8b2cb51f8e7..ba4be9f9b20 100644 --- a/spec/unit/matrix-client.spec.ts +++ b/spec/unit/matrix-client.spec.ts @@ -1703,4 +1703,39 @@ describe("MatrixClient", function() { expect(newSourceRoom._state.get(PolicyScope.User)?.[eventId]).toBeFalsy(); }); }); + + describe("using E2EE in group calls", () => { + const opts = { + baseUrl: "https://my.home.server", + idBaseUrl: identityServerUrl, + accessToken: "my.access.token", + store: store, + scheduler: scheduler, + userId: userId, + }; + + it("enables E2EE by default", () => { + const client = new MatrixClient(opts); + + expect(client.getUseE2eForGroupCall()).toBe(true); + }); + + it("enables E2EE when enabled explicitly", () => { + const client = new MatrixClient({ + useE2eForGroupCall: true, + ...opts, + }); + + expect(client.getUseE2eForGroupCall()).toBe(true); + }); + + it("disables E2EE if disabled explicitly", () => { + const client = new MatrixClient({ + useE2eForGroupCall: false, + ...opts, + }); + + expect(client.getUseE2eForGroupCall()).toBe(false); + }); + }); }); diff --git a/spec/unit/queueToDevice.spec.ts b/spec/unit/queueToDevice.spec.ts index afdd7fdd55b..298f5341ca9 100644 --- a/spec/unit/queueToDevice.spec.ts +++ b/spec/unit/queueToDevice.spec.ts @@ -22,6 +22,7 @@ import { MatrixClient } from "../../src/client"; import { ToDeviceBatch } from '../../src/models/ToDeviceMessage'; import { logger } from '../../src/logger'; import { IStore } from '../../src/store'; +import { flushPromises } from '../test-utils/flushPromises'; import { removeElement } from "../../src/utils"; const FAKE_USER = "@alice:example.org"; @@ -48,19 +49,6 @@ enum StoreType { IndexedDB = 'IndexedDB', } -// Jest now uses @sinonjs/fake-timers which exposes tickAsync() and a number of -// other async methods which break the event loop, letting scheduled promise -// callbacks run. Unfortunately, Jest doesn't expose these, so we have to do -// it manually (this is what sinon does under the hood). We do both in a loop -// until the thing we expect happens: hopefully this is the least flakey way -// and avoids assuming anything about the app's behaviour. -const realSetTimeout = setTimeout; -function flushPromises() { - return new Promise(r => { - realSetTimeout(r, 1); - }); -} - async function flushAndRunTimersUntil(cond: () => boolean) { while (!cond()) { await flushPromises(); diff --git a/spec/unit/webrtc/call.spec.ts b/spec/unit/webrtc/call.spec.ts index 8b42d9fe61a..030e6a6ba68 100644 --- a/spec/unit/webrtc/call.spec.ts +++ b/spec/unit/webrtc/call.spec.ts @@ -14,192 +14,230 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { mocked } from "jest-mock"; + import { TestClient } from '../../TestClient'; -import { MatrixCall, CallErrorCode, CallEvent, supportsMatrixCall, CallType } from '../../../src/webrtc/call'; -import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from '../../../src/webrtc/callEventTypes'; +import { + MatrixCall, + CallErrorCode, + CallEvent, + supportsMatrixCall, + CallType, + CallState, + CallParty, +} from '../../../src/webrtc/call'; +import { + MCallAnswer, + MCallHangupReject, + SDPStreamMetadata, + SDPStreamMetadataKey, + SDPStreamMetadataPurpose, +} from '../../../src/webrtc/callEventTypes'; import { DUMMY_SDP, MockMediaHandler, MockMediaStream, MockMediaStreamTrack, - MockMediaDeviceInfo, + installWebRTCMocks, MockRTCPeerConnection, + SCREENSHARE_STREAM_ID, } from "../../test-utils/webrtc"; import { CallFeed } from "../../../src/webrtc/callFeed"; -import { EventType, MatrixClient } from "../../../src"; -import { MediaHandler } from "../../../src/webrtc/mediaHandler"; +import { EventType, IContent, ISendEventResponse, MatrixEvent, Room } from "../../../src"; + +const FAKE_ROOM_ID = "!foo:bar"; +const CALL_LIFETIME = 60000; -const startVoiceCall = async (client: TestClient, call: MatrixCall): Promise => { +const startVoiceCall = async (client: TestClient, call: MatrixCall, userId?: string): Promise => { const callPromise = call.placeVoiceCall(); await client.httpBackend!.flush(""); await callPromise; - call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); + call.getOpponentMember = jest.fn().mockReturnValue({ userId: userId ?? "@bob:bar.uk" }); +}; + +const startVideoCall = async (client: TestClient, call: MatrixCall, userId?: string): Promise => { + const callPromise = call.placeVideoCall(); + await client.httpBackend.flush(""); + await callPromise; + + call.getOpponentMember = jest.fn().mockReturnValue({ userId: userId ?? "@bob:bar.uk" }); +}; + +const fakeIncomingCall = async (client: TestClient, call: MatrixCall, version: string | number = "1") => { + const callPromise = call.initWithInvite({ + getContent: jest.fn().mockReturnValue({ + version, + call_id: "call_id", + party_id: "remote_party_id", + lifetime: CALL_LIFETIME, + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + getLocalAge: () => 1, + } as unknown as MatrixEvent); + call.getFeeds().push(new CallFeed({ + client: client.client, + userId: "remote_user_id", + // @ts-ignore Mock + stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), + id: "remote_feed_id", + purpose: SDPStreamMetadataPurpose.Usermedia, + })); + await callPromise; }; +function makeMockEvent(sender: string, content: Record): MatrixEvent { + return { + getContent: () => { + return content; + }, + getSender: () => sender, + } as MatrixEvent; +} + describe('Call', function() { let client: TestClient; - let call; - let prevNavigator; - let prevDocument; - let prevWindow; + let call: MatrixCall; + let prevNavigator: Navigator; + let prevDocument: Document; + let prevWindow: Window & typeof globalThis; + // We retain a reference to this in the correct Mock type + let mockSendEvent: jest.Mock, [string, string, IContent, string]>; + + const errorListener = () => {}; beforeEach(function() { prevNavigator = global.navigator; prevDocument = global.document; prevWindow = global.window; - global.navigator = { - mediaDevices: { - // @ts-ignore Mock - getUserMedia: () => new MockMediaStream("local_stream"), - // @ts-ignore Mock - enumerateDevices: async () => [new MockMediaDeviceInfo("audio"), new MockMediaDeviceInfo("video")], - }, - }; - - global.window = { - // @ts-ignore Mock - RTCPeerConnection: MockRTCPeerConnection, - // @ts-ignore Mock - RTCSessionDescription: {}, - // @ts-ignore Mock - RTCIceCandidate: {}, - getUserMedia: () => new MockMediaStream("local_stream"), - }; - // @ts-ignore Mock - global.document = {}; + installWebRTCMocks(); client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}); // We just stub out sendEvent: we're not interested in testing the client's // event sending code here - client.client.sendEvent = (() => {}) as unknown as MatrixClient["sendEvent"]; - client.client["mediaHandler"] = new MockMediaHandler as unknown as MediaHandler; - client.client.getMediaHandler = () => client.client["mediaHandler"]!; - client.httpBackend!.when("GET", "/voip/turnServer").respond(200, {}); + client.client.sendEvent = mockSendEvent = jest.fn(); + { + // in which we do naughty assignments to private members + const untypedClient = (client.client as any); + untypedClient.mediaHandler = new MockMediaHandler; + untypedClient.turnServersExpiry = Date.now() + 60 * 60 * 1000; + } + + client.httpBackend.when("GET", "/voip/turnServer").respond(200, {}); + client.client.getRoom = () => { + return { + getMember: () => { + return {}; + }, + } as unknown as Room; + }; + client.client.getProfileInfo = jest.fn(); + call = new MatrixCall({ client: client.client, - roomId: '!foo:bar', + roomId: FAKE_ROOM_ID, }); // call checks one of these is wired up - call.on('error', () => {}); + call.on(CallEvent.Error, errorListener); }); afterEach(function() { + // Hangup to stop timers + call.hangup(CallErrorCode.UserHangup, true); + client.stop(); global.navigator = prevNavigator; global.window = prevWindow; global.document = prevDocument; + + jest.useRealTimers(); }); it('should ignore candidate events from non-matching party ID', async function() { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + answer: { + sdp: DUMMY_SDP, }, - }); - - call.peerConn.addIceCandidate = jest.fn(); - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - candidates: [ - { - candidate: '', - sdpMid: '', - }, - ], - }; - }, - }); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); - - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'some_other_party_id', - candidates: [ - { - candidate: '', - sdpMid: '', - }, - ], - }; - }, - }); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); + })); - // Hangup to stop timers - call.hangup(CallErrorCode.UserHangup, true); + const mockAddIceCandidate = call.peerConn!.addIceCandidate = jest.fn(); + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + candidates: [ + { + candidate: '', + sdpMid: '', + }, + ], + })); + expect(mockAddIceCandidate).toHaveBeenCalled(); + + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'some_other_party_id', + candidates: [ + { + candidate: '', + sdpMid: '', + }, + ], + })); + expect(mockAddIceCandidate).toHaveBeenCalled(); }); it('should add candidates received before answer if party ID is correct', async function() { await startVoiceCall(client, call); - call.peerConn.addIceCandidate = jest.fn(); - - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - candidates: [ - { - candidate: 'the_correct_candidate', - sdpMid: '', - }, - ], - }; - }, - }); + const mockAddIceCandidate = call.peerConn!.addIceCandidate = jest.fn(); + + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + candidates: [ + { + candidate: 'the_correct_candidate', + sdpMid: '', + }, + ], + })); - call.onRemoteIceCandidatesReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'some_other_party_id', - candidates: [ - { - candidate: 'the_wrong_candidate', - sdpMid: '', - }, - ], - }; - }, - }); + call.onRemoteIceCandidatesReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'some_other_party_id', + candidates: [ + { + candidate: 'the_wrong_candidate', + sdpMid: '', + }, + ], + })); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(0); + expect(mockAddIceCandidate).not.toHaveBeenCalled(); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'the_correct_party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + })); - expect(call.peerConn.addIceCandidate.mock.calls.length).toBe(1); - expect(call.peerConn.addIceCandidate).toHaveBeenCalledWith({ + expect(mockAddIceCandidate).toHaveBeenCalled(); + expect(mockAddIceCandidate).toHaveBeenCalledWith({ candidate: 'the_correct_candidate', sdpMid: '', }); @@ -207,70 +245,55 @@ describe('Call', function() { it('should map asserted identity messages to remoteAssertedIdentity', async function() { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + })); const identChangedCallback = jest.fn(); call.on(CallEvent.AssertedIdentityChanged, identChangedCallback); - await call.onAssertedIdentityReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - asserted_identity: { - id: "@steve:example.com", - display_name: "Steve Gibbons", - }, - }; + await call.onAssertedIdentityReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + asserted_identity: { + id: "@steve:example.com", + display_name: "Steve Gibbons", }, - }); + })); expect(identChangedCallback).toHaveBeenCalled(); const ident = call.getRemoteAssertedIdentity()!; expect(ident.id).toEqual("@steve:example.com"); expect(ident.displayName).toEqual("Steve Gibbons"); - - // Hangup to stop timers - call.hangup(CallErrorCode.UserHangup, true); }); it("should map SDPStreamMetadata to feeds", async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: { - "remote_stream": { - purpose: SDPStreamMetadataPurpose.Usermedia, - audio_muted: true, - video_muted: false, - }, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + [SDPStreamMetadataKey]: { + "remote_stream": { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, + }, + }, + })); - call.pushRemoteFeed( + (call as any).pushRemoteFeed( new MockMediaStream( "remote_stream", [ @@ -288,39 +311,35 @@ describe('Call', function() { it("should fallback to replaceTrack() if the other side doesn't support SPDStreamMetadata", async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + })); - call.setScreensharingEnabledWithoutMetadataSupport = jest.fn(); + const mockScreenshareNoMetadata = (call as any).setScreensharingEnabledWithoutMetadataSupport = jest.fn(); call.setScreensharingEnabled(true); - expect(call.setScreensharingEnabledWithoutMetadataSupport).toHaveBeenCalled(); + expect(mockScreenshareNoMetadata).toHaveBeenCalled(); }); it("should fallback to answering with no video", async () => { - await client.httpBackend!.flush(undefined); + await client.httpBackend!.flush(""); - call.shouldAnswerWithMediaType = (wantedValue: boolean) => wantedValue; - client.client["mediaHandler"].getUserMediaStream = jest.fn().mockRejectedValue("reject"); + (call as any).shouldAnswerWithMediaType = (wantedValue: boolean) => wantedValue; + client.client.getMediaHandler().getUserMediaStream = jest.fn().mockRejectedValue("reject"); await call.answer(true, true); - expect(client.client["mediaHandler"].getUserMediaStream).toHaveBeenNthCalledWith(1, true, true); - expect(client.client["mediaHandler"].getUserMediaStream).toHaveBeenNthCalledWith(2, true, false); + expect(client.client.getMediaHandler().getUserMediaStream).toHaveBeenNthCalledWith(1, true, true); + expect(client.client.getMediaHandler().getUserMediaStream).toHaveBeenNthCalledWith(2, true, false); }); it("should handle mid-call device changes", async () => { - client.client["mediaHandler"].getUserMediaStream = jest.fn().mockReturnValue( + client.client.getMediaHandler().getUserMediaStream = jest.fn().mockReturnValue( new MockMediaStream( "stream", [ new MockMediaStreamTrack("audio_track", "audio"), @@ -331,18 +350,14 @@ describe('Call', function() { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + })); await call.updateLocalUsermediaStream( new MockMediaStream( @@ -351,127 +366,150 @@ describe('Call', function() { new MockMediaStreamTrack("new_audio_track", "audio"), new MockMediaStreamTrack("video_track", "video"), ], - ), + ).typed(), ); - expect(call.localUsermediaStream.id).toBe("stream"); - expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("new_audio_track"); - expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); - expect(call.usermediaSenders.find((sender) => { - return sender?.track?.kind === "audio"; - }).track.id).toBe("new_audio_track"); - expect(call.usermediaSenders.find((sender) => { - return sender?.track?.kind === "video"; - }).track.id).toBe("video_track"); + + // XXX: Lots of inspecting the prvate state of the call object here + const transceivers: Map = (call as any).transceivers; + + expect(call.localUsermediaStream!.id).toBe("stream"); + expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("new_audio_track"); + expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("video_track"); + // call has a function for generating these but we hardcode here to avoid exporting it + expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("new_audio_track"); + expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("video_track"); }); it("should handle upgrade to video call", async () => { await startVoiceCall(client, call); - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: {}, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + [SDPStreamMetadataKey]: {}, + })); + + // XXX Should probably test using the public interfaces, ie. + // setLocalVideoMuted probably? + await (call as any).upgradeCall(false, true); - await call.upgradeCall(false, true); + // XXX: More inspecting private state of the call object + const transceivers: Map = (call as any).transceivers; - expect(call.localUsermediaStream.getAudioTracks()[0].id).toBe("audio_track"); - expect(call.localUsermediaStream.getVideoTracks()[0].id).toBe("video_track"); - expect(call.usermediaSenders.find((sender) => { - return sender?.track?.kind === "audio"; - }).track.id).toBe("audio_track"); - expect(call.usermediaSenders.find((sender) => { - return sender?.track?.kind === "video"; - }).track.id).toBe("video_track"); + expect(call.localUsermediaStream!.getAudioTracks()[0].id).toBe("usermedia_audio_track"); + expect(call.localUsermediaStream!.getVideoTracks()[0].id).toBe("usermedia_video_track"); + expect(transceivers.get("m.usermedia:audio")!.sender.track!.id).toBe("usermedia_audio_track"); + expect(transceivers.get("m.usermedia:video")!.sender.track!.id).toBe("usermedia_video_track"); }); it("should handle SDPStreamMetadata changes", async () => { await startVoiceCall(client, call); - call.updateRemoteSDPStreamMetadata({ + (call as any).updateRemoteSDPStreamMetadata({ "remote_stream": { purpose: SDPStreamMetadataPurpose.Usermedia, audio_muted: false, video_muted: false, }, }); - call.pushRemoteFeed(new MockMediaStream("remote_stream", [])); + (call as any).pushRemoteFeed(new MockMediaStream("remote_stream", [])); const feed = call.getFeeds().find((feed) => feed.stream.id === "remote_stream"); - call.onSDPStreamMetadataChangedReceived({ - getContent: () => ({ - [SDPStreamMetadataKey]: { - "remote_stream": { - purpose: SDPStreamMetadataPurpose.Screenshare, - audio_muted: true, - video_muted: true, - id: "feed_id2", - }, + call.onSDPStreamMetadataChangedReceived(makeMockEvent("@test:foo", { + [SDPStreamMetadataKey]: { + "remote_stream": { + purpose: SDPStreamMetadataPurpose.Screenshare, + audio_muted: true, + video_muted: true, + id: "feed_id2", }, - }), - }); + }, + })); expect(feed?.purpose).toBe(SDPStreamMetadataPurpose.Screenshare); - expect(feed?.audioMuted).toBe(true); - expect(feed?.videoMuted).toBe(true); + expect(feed?.isAudioMuted()).toBe(true); + expect(feed?.isVideoMuted()).toBe(true); }); it("should choose opponent member", async () => { const callPromise = call.placeVoiceCall(); - await client.httpBackend!.flush(undefined); + await client.httpBackend!.flush(""); await callPromise; const opponentMember = { roomId: call.roomId, userId: "opponentUserId", }; + + client.client.getRoom = () => { + return { + getMember: (userId) => { + if (userId === opponentMember.userId) { + return opponentMember; + } + }, + } as unknown as Room; + }; + const opponentCaps = { "m.call.transferee": true, "m.call.dtmf": false, }; - call.chooseOpponent({ - getContent: () => ({ - version: 1, - party_id: "party_id", - capabilities: opponentCaps, - }), - sender: opponentMember, - }); + (call as any).chooseOpponent(makeMockEvent(opponentMember.userId, { + version: 1, + party_id: "party_id", + capabilities: opponentCaps, + })); expect(call.getOpponentMember()).toBe(opponentMember); - expect(call.opponentPartyId).toBe("party_id"); - expect(call.opponentCaps).toBe(opponentCaps); + expect((call as any).opponentPartyId).toBe("party_id"); + expect((call as any).opponentCaps).toBe(opponentCaps); expect(call.opponentCanBeTransferred()).toBe(true); expect(call.opponentSupportsDTMF()).toBe(false); }); describe("should deduce the call type correctly", () => { + beforeEach(async () => { + // start an incoming call, but add no feeds + await call.initWithInvite({ + getContent: jest.fn().mockReturnValue({ + version: "1", + call_id: "call_id", + party_id: "remote_party_id", + lifetime: CALL_LIFETIME, + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + getLocalAge: () => 1, + } as unknown as MatrixEvent); + }); + it("if no video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [])); + (call as any).pushRemoteFeed(new MockMediaStream("remote_stream1", [])); expect(call.type).toBe(CallType.Voice); }); it("if remote video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushRemoteFeed(new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")])); + (call as any).pushRemoteFeed( + new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), + ); expect(call.type).toBe(CallType.Video); }); it("if local video", async () => { call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushNewLocalFeed( + (call as any).pushNewLocalFeed( new MockMediaStream("remote_stream1", [new MockMediaStreamTrack("track_id", "video")]), SDPStreamMetadataPurpose.Usermedia, false, @@ -491,17 +529,17 @@ describe('Call', function() { audioMuted: false, videoMuted: false, })]); - await client.httpBackend!.flush(undefined); + await client.httpBackend!.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.pushNewLocalFeed( + (call as any).pushNewLocalFeed( new MockMediaStream("local_stream2", [new MockMediaStreamTrack("track_id", "video")]), SDPStreamMetadataPurpose.Screenshare, "feed_id2", ); await call.setMicrophoneMuted(true); - expect(call.getLocalSDPStreamMetadata()).toStrictEqual({ + expect((call as any).getLocalSDPStreamMetadata()).toStrictEqual({ "local_stream1": { "purpose": SDPStreamMetadataPurpose.Usermedia, "audio_muted": true, @@ -543,14 +581,13 @@ describe('Call', function() { videoMuted: false, }), ]); - await client.httpBackend!.flush(undefined); + await client.httpBackend!.flush(""); await callPromise; call.getOpponentMember = jest.fn().mockReturnValue({ userId: "@bob:bar.uk" }); - call.updateRemoteSDPStreamMetadata({ + (call as any).updateRemoteSDPStreamMetadata({ "remote_usermedia_stream_id": { purpose: SDPStreamMetadataPurpose.Usermedia, - id: "remote_usermedia_feed_id", audio_muted: false, video_muted: false, }, @@ -561,54 +598,32 @@ describe('Call', function() { video_muted: false, }, }); - call.pushRemoteFeed(remoteUsermediaStream); - call.pushRemoteFeed(remoteScreensharingStream); + (call as any).pushRemoteFeed(remoteUsermediaStream); + (call as any).pushRemoteFeed(remoteScreensharingStream); - expect(call.localUsermediaFeed.stream).toBe(localUsermediaStream); + expect(call.localUsermediaFeed!.stream).toBe(localUsermediaStream); expect(call.localUsermediaStream).toBe(localUsermediaStream); - expect(call.localScreensharingFeed.stream).toBe(localScreensharingStream); + expect(call.localScreensharingFeed!.stream).toBe(localScreensharingStream); expect(call.localScreensharingStream).toBe(localScreensharingStream); - expect(call.remoteUsermediaFeed.stream).toBe(remoteUsermediaStream); + expect(call.remoteUsermediaFeed!.stream).toBe(remoteUsermediaStream); expect(call.remoteUsermediaStream).toBe(remoteUsermediaStream); - expect(call.remoteScreensharingFeed.stream).toBe(remoteScreensharingStream); + expect(call.remoteScreensharingFeed!.stream).toBe(remoteScreensharingStream); expect(call.remoteScreensharingStream).toBe(remoteScreensharingStream); expect(call.hasRemoteUserMediaAudioTrack).toBe(false); }); it("should end call after receiving a select event with a different party id", async () => { - const callPromise = call.initWithInvite({ - getContent: () => ({ - version: 1, - call_id: "call_id", - party_id: "remote_party_id", - offer: { - sdp: DUMMY_SDP, - }, - }), - getLocalAge: () => null, - }); - call.feeds.push(new CallFeed({ - client: client.client, - userId: "remote_user_id", - // @ts-ignore Mock - stream: new MockMediaStream("remote_stream_id", [new MockMediaStreamTrack("remote_tack_id")]), - id: "remote_feed_id", - purpose: SDPStreamMetadataPurpose.Usermedia, - })); - await client.httpBackend!.flush(undefined); - await callPromise; + await fakeIncomingCall(client, call); const callHangupCallback = jest.fn(); call.on(CallEvent.Hangup, callHangupCallback); - await call.onSelectAnswerReceived({ - getContent: () => ({ - version: 1, - call_id: call.callId, - party_id: 'party_id', - selected_party_id: "different_party_id", - }), - }); + await call.onSelectAnswerReceived(makeMockEvent("@test:foo.bar", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + selected_party_id: "different_party_id", + })); expect(callHangupCallback).toHaveBeenCalled(); }); @@ -698,55 +713,53 @@ describe('Call', function() { describe("ignoring streams with ids for which we already have a feed", () => { const STREAM_ID = "stream_id"; - const FEEDS_CHANGED_CALLBACK = jest.fn(); + let FEEDS_CHANGED_CALLBACK: jest.Mock; beforeEach(async () => { + FEEDS_CHANGED_CALLBACK = jest.fn(); + await startVoiceCall(client, call); call.on(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); jest.spyOn(call, "pushLocalFeed"); }); afterEach(() => { - FEEDS_CHANGED_CALLBACK.mockReset(); + call.off(CallEvent.FeedsChanged, FEEDS_CHANGED_CALLBACK); }); it("should ignore stream passed to pushRemoteFeed()", async () => { - await call.onAnswerReceived({ - getContent: () => { - return { - version: 1, - call_id: call.callId, - party_id: 'party_id', - answer: { - sdp: DUMMY_SDP, - }, - [SDPStreamMetadataKey]: { - [STREAM_ID]: { - purpose: SDPStreamMetadataPurpose.Usermedia, - }, - }, - }; + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'party_id', + answer: { + sdp: DUMMY_SDP, }, - }); + [SDPStreamMetadataKey]: { + [STREAM_ID]: { + purpose: SDPStreamMetadataPurpose.Usermedia, + }, + }, + })); - call.pushRemoteFeed(new MockMediaStream(STREAM_ID)); - call.pushRemoteFeed(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeed(new MockMediaStream(STREAM_ID)); expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); it("should ignore stream passed to pushRemoteFeedWithoutMetadata()", async () => { - call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); - call.pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); + (call as any).pushRemoteFeedWithoutMetadata(new MockMediaStream(STREAM_ID)); expect(call.getRemoteFeeds().length).toBe(1); expect(FEEDS_CHANGED_CALLBACK).toHaveBeenCalledTimes(1); }); it("should ignore stream passed to pushNewLocalFeed()", async () => { - call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); - call.pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); + (call as any).pushNewLocalFeed(new MockMediaStream(STREAM_ID), SDPStreamMetadataPurpose.Screenshare); // We already have one local feed from placeVoiceCall() expect(call.getLocalFeeds().length).toBe(2); @@ -767,4 +780,677 @@ describe('Call', function() { })); }); }); + + describe("muting", () => { + let mockSendVoipEvent: jest.Mock, [string, object]>; + beforeEach(async () => { + (call as any).sendVoipEvent = mockSendVoipEvent = jest.fn(); + await startVideoCall(client, call); + }); + + describe("sending sdp_stream_metadata_changed events", () => { + it("should send sdp_stream_metadata_changed when muting audio", async () => { + await call.setMicrophoneMuted(true); + expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: { + mock_stream_from_media_handler: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: true, + video_muted: false, + }, + }, + }); + }); + + it("should send sdp_stream_metadata_changed when muting video", async () => { + await call.setLocalVideoMuted(true); + expect(mockSendVoipEvent).toHaveBeenCalledWith(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: { + mock_stream_from_media_handler: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: false, + video_muted: true, + }, + }, + }); + }); + }); + + describe("receiving sdp_stream_metadata_changed events", () => { + const setupCall = (audio: boolean, video: boolean): SDPStreamMetadata => { + const metadata = { + stream: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }; + (call as any).pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("track1", "audio"), + new MockMediaStreamTrack("track1", "video"), + ])); + call.onSDPStreamMetadataChangedReceived({ + getContent: () => ({ + [SDPStreamMetadataKey]: metadata, + }), + } as MatrixEvent); + return metadata; + }; + + it("should handle incoming sdp_stream_metadata_changed with audio muted", async () => { + const metadata = setupCall(true, false); + expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); + expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(true); + expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(false); + }); + + it("should handle incoming sdp_stream_metadata_changed with video muted", async () => { + const metadata = setupCall(false, true); + expect((call as any).remoteSDPStreamMetadata).toStrictEqual(metadata); + expect(call.getRemoteFeeds()[0].isAudioMuted()).toBe(false); + expect(call.getRemoteFeeds()[0].isVideoMuted()).toBe(true); + }); + }); + }); + + describe("rejecting calls", () => { + it("sends hangup event when rejecting v0 calls", async () => { + await fakeIncomingCall(client, call, 0); + + call.reject(); + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallHangup, + expect.objectContaining({ + call_id: call.callId, + }), + ); + }); + + it("sends reject event when rejecting v1 calls", async () => { + await fakeIncomingCall(client, call, "1"); + + call.reject(); + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallReject, + expect.objectContaining({ + call_id: call.callId, + }), + ); + }); + + it("does not reject a call that has already been answered", async () => { + await fakeIncomingCall(client, call, "1"); + + await call.answer(); + + mockSendEvent.mockReset(); + + let caught = false; + try { + call.reject(); + } catch (e) { + caught = true; + } + + expect(caught).toEqual(true); + expect(client.client.sendEvent).not.toHaveBeenCalled(); + + call.hangup(CallErrorCode.UserHangup, true); + }); + + it("hangs up a call", async () => { + await fakeIncomingCall(client, call, "1"); + + await call.answer(); + + mockSendEvent.mockReset(); + + call.hangup(CallErrorCode.UserHangup, true); + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallHangup, + expect.objectContaining({ + call_id: call.callId, + }), + ); + }); + }); + + describe("answering calls", () => { + const realSetTimeout = setTimeout; + + beforeEach(async () => { + await fakeIncomingCall(client, call, "1"); + }); + + const untilEventSent = async (...args) => { + const maxTries = 20; + + for (let tries = 0; tries < maxTries; ++tries) { + if (tries) { + await new Promise(resolve => { + realSetTimeout(resolve, 100); + }); + } + // We might not always be in fake timer mode, but it's + // fine to run this if not, so we just call it anyway. + jest.runOnlyPendingTimers(); + try { + expect(mockSendEvent).toHaveBeenCalledWith(...args); + return; + } catch (e) { + if (tries == maxTries - 1) { + throw e; + } + } + } + }; + + it("sends an answer event", async () => { + await call.answer(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallAnswer, + expect.objectContaining({ + call_id: call.callId, + answer: expect.objectContaining({ + type: 'offer', + }), + }), + ); + }); + + describe("ICE candidate sending", () => { + let mockPeerConn; + const fakeCandidateString = "here is a fake candidate!"; + const fakeCandidateEvent = { + candidate: { + candidate: fakeCandidateString, + sdpMLineIndex: 0, + sdpMid: '0', + toJSON: jest.fn().mockReturnValue(fakeCandidateString), + }, + } as unknown as RTCPeerConnectionIceEvent; + + beforeEach(async () => { + await call.answer(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallAnswer, + expect.objectContaining({}), + ); + mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it("sends ICE candidates as separate events if they arrive after the answer", async () => { + mockPeerConn!.iceCandidateListener!(fakeCandidateEvent); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + }); + + it("retries sending ICE candidates", async () => { + jest.useFakeTimers(); + + mockSendEvent.mockRejectedValueOnce(new Error("Fake error")); + + mockPeerConn!.iceCandidateListener!(fakeCandidateEvent); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + + mockSendEvent.mockClear(); + + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + }); + + it("gives up on call after 5 attempts at sending ICE candidates", async () => { + jest.useFakeTimers(); + + mockSendEvent.mockImplementation((roomId: string, eventType: string) => { + if (eventType === EventType.CallCandidates) { + return Promise.reject(new Error()); + } else { + return Promise.resolve({ event_id: "foo" }); + } + }); + + mockPeerConn!.iceCandidateListener!(fakeCandidateEvent); + + while (!call.callHasEnded()) { + jest.runOnlyPendingTimers(); + await untilEventSent( + FAKE_ROOM_ID, + EventType.CallCandidates, + expect.objectContaining({ + candidates: [ + fakeCandidateString, + ], + }), + ); + if (!call.callHasEnded) { + mockSendEvent.mockReset(); + } + } + + expect(call.callHasEnded()).toEqual(true); + }); + }); + }); + + it("times out an incoming call", async () => { + jest.useFakeTimers(); + await fakeIncomingCall(client, call, "1"); + + expect(call.state).toEqual(CallState.Ringing); + + jest.advanceTimersByTime(CALL_LIFETIME + 1000); + + expect(call.state).toEqual(CallState.Ended); + }); + + describe("Screen sharing", () => { + const waitNegotiateFunc = resolve => { + mockSendEvent.mockImplementationOnce(() => { + // Note that the peer connection here is a dummy one and always returns + // dummy SDP, so there's not much point returning the content: the SDP will + // always be the same. + resolve(); + return Promise.resolve({ event_id: "foo" }); + }); + }; + + beforeEach(async () => { + await startVoiceCall(client, call); + + const sendNegotiatePromise = new Promise(waitNegotiateFunc); + + MockRTCPeerConnection.triggerAllNegotiations(); + await sendNegotiatePromise; + + await call.onAnswerReceived(makeMockEvent("@test:foo", { + "version": 1, + "call_id": call.callId, + "party_id": 'party_id', + "answer": { + sdp: DUMMY_SDP, + }, + "org.matrix.msc3077.sdp_stream_metadata": { + "foo": { + "purpose": "m.usermedia", + "audio_muted": false, + "video_muted": false, + }, + }, + })); + }); + + afterEach(() => { + // Hangup to stop timers + call.hangup(CallErrorCode.UserHangup, true); + }); + + it("enables and disables screensharing", async () => { + await call.setScreensharingEnabled(true); + + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), + ).toHaveLength(1); + + mockSendEvent.mockReset(); + const sendNegotiatePromise = new Promise(waitNegotiateFunc); + + MockRTCPeerConnection.triggerAllNegotiations(); + await sendNegotiatePromise; + + expect(client.client.sendEvent).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.CallNegotiate, + expect.objectContaining({ + "version": "1", + "call_id": call.callId, + "org.matrix.msc3077.sdp_stream_metadata": expect.objectContaining({ + [SCREENSHARE_STREAM_ID]: expect.objectContaining({ + purpose: SDPStreamMetadataPurpose.Screenshare, + }), + }), + }), + ); + + await call.setScreensharingEnabled(false); + + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), + ).toHaveLength(0); + }); + + it("removes RTX codec from screen sharing transcievers", async () => { + mocked(global.RTCRtpSender.getCapabilities).mockReturnValue({ + codecs: [ + { mimeType: "video/rtx", clockRate: 90000 }, + { mimeType: "video/somethingelse", clockRate: 90000 }, + ], + headerExtensions: [], + }); + + mockSendEvent.mockReset(); + const sendNegotiatePromise = new Promise(waitNegotiateFunc); + + await call.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + + await sendNegotiatePromise; + + const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + expect( + mockPeerConn.transceivers[mockPeerConn.transceivers.length - 1].setCodecPreferences, + ).toHaveBeenCalledWith([expect.objectContaining({ mimeType: "video/somethingelse" })]); + }); + + it("re-uses transceiver when screen sharing is re-enabled", async () => { + const mockPeerConn = call.peerConn as unknown as MockRTCPeerConnection; + + // sanity check: we should start with one transciever (user media audio) + expect(mockPeerConn.transceivers.length).toEqual(1); + + const screenshareOnProm1 = new Promise(waitNegotiateFunc); + + await call.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + + await screenshareOnProm1; + + // we should now have another transciever for the screenshare + expect(mockPeerConn.transceivers.length).toEqual(2); + + const screenshareOffProm = new Promise(waitNegotiateFunc); + await call.setScreensharingEnabled(false); + MockRTCPeerConnection.triggerAllNegotiations(); + await screenshareOffProm; + + // both transceivers should still be there + expect(mockPeerConn.transceivers.length).toEqual(2); + + const screenshareOnProm2 = new Promise(waitNegotiateFunc); + await call.setScreensharingEnabled(true); + MockRTCPeerConnection.triggerAllNegotiations(); + await screenshareOnProm2; + + // should still be two, ie. another one should not have been created + // when re-enabling the screen share. + expect(mockPeerConn.transceivers.length).toEqual(2); + }); + }); + + it("falls back to replaceTrack for opponents that don't support stream metadata", async () => { + await startVideoCall(client, call); + + await call.onAnswerReceived(makeMockEvent("@test:foo", { + "version": 1, + "call_id": call.callId, + "party_id": 'party_id', + "answer": { + sdp: DUMMY_SDP, + }, + })); + + MockRTCPeerConnection.triggerAllNegotiations(); + + const mockVideoSender = call.peerConn!.getSenders().find(s => s.track!.kind === "video"); + const mockReplaceTrack = mockVideoSender!.replaceTrack = jest.fn(); + + await call.setScreensharingEnabled(true); + + // our local feed should still reflect the purpose of the feed (ie. screenshare) + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare).length, + ).toEqual(1); + + // but we should not have re-negotiated + expect(MockRTCPeerConnection.hasAnyPendingNegotiations()).toEqual(false); + + expect(mockReplaceTrack).toHaveBeenCalledWith(expect.objectContaining({ + id: "screenshare_video_track", + })); + mockReplaceTrack.mockClear(); + + await call.setScreensharingEnabled(false); + + expect( + call.getLocalFeeds().filter(f => f.purpose == SDPStreamMetadataPurpose.Screenshare), + ).toHaveLength(0); + expect(call.getLocalFeeds()).toHaveLength(1); + + expect(MockRTCPeerConnection.hasAnyPendingNegotiations()).toEqual(false); + + expect(mockReplaceTrack).toHaveBeenCalledWith(expect.objectContaining({ + id: "usermedia_video_track", + })); + }); + + describe("call transfers", () => { + const ALICE_USER_ID = "@alice:foo"; + const ALICE_DISPLAY_NAME = "Alice"; + const ALICE_AVATAR_URL = "avatar.alice.foo"; + + const BOB_USER_ID = "@bob:foo"; + const BOB_DISPLAY_NAME = "Bob"; + const BOB_AVATAR_URL = "avatar.bob.foo"; + + beforeEach(() => { + mocked(client.client.getProfileInfo).mockImplementation(async (userId) => { + if (userId === ALICE_USER_ID) { + return { + displayname: ALICE_DISPLAY_NAME, + avatar_url: ALICE_AVATAR_URL, + }; + } else if (userId === BOB_USER_ID) { + return { + displayname: BOB_DISPLAY_NAME, + avatar_url: BOB_AVATAR_URL, + }; + } else { + return {}; + } + }); + }); + + it("transfers call to another call", async () => { + const newCall = new MatrixCall({ + client: client.client, + roomId: FAKE_ROOM_ID, + }); + + const callHangupListener = jest.fn(); + const newCallHangupListener = jest.fn(); + + call.on(CallEvent.Hangup, callHangupListener); + newCall.on(CallEvent.Error, () => { }); + newCall.on(CallEvent.Hangup, newCallHangupListener); + + await startVoiceCall(client, call, ALICE_USER_ID); + await startVoiceCall(client, newCall, BOB_USER_ID); + + await call.transferToCall(newCall); + + expect(mockSendEvent).toHaveBeenCalledWith(FAKE_ROOM_ID, EventType.CallReplaces, expect.objectContaining({ + target_user: { + id: ALICE_USER_ID, + display_name: ALICE_DISPLAY_NAME, + avatar_url: ALICE_AVATAR_URL, + }, + })); + expect(mockSendEvent).toHaveBeenCalledWith(FAKE_ROOM_ID, EventType.CallReplaces, expect.objectContaining({ + target_user: { + id: BOB_USER_ID, + display_name: BOB_DISPLAY_NAME, + avatar_url: BOB_AVATAR_URL, + }, + })); + + expect(callHangupListener).toHaveBeenCalledWith(call); + expect(newCallHangupListener).toHaveBeenCalledWith(newCall); + }); + + it("transfers a call to another user", async () => { + // @ts-ignore Mock + jest.spyOn(call, "terminate"); + + await startVoiceCall(client, call, ALICE_USER_ID); + await call.transfer(BOB_USER_ID); + + expect(mockSendEvent).toHaveBeenCalledWith(FAKE_ROOM_ID, EventType.CallReplaces, expect.objectContaining({ + target_user: { + id: BOB_USER_ID, + display_name: BOB_DISPLAY_NAME, + avatar_url: BOB_AVATAR_URL, + }, + })); + // @ts-ignore Mock + expect(call.terminate).toHaveBeenCalledWith(CallParty.Local, CallErrorCode.Transfered, true); + }); + }); + + describe("onTrack", () => { + it("ignores streamless track", async () => { + // @ts-ignore Mock pushRemoteFeed() is private + jest.spyOn(call, "pushRemoteFeed"); + + await call.placeVoiceCall(); + + (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!( + { streams: [], track: new MockMediaStreamTrack("track_ev", "audio") } as unknown as RTCTrackEvent, + ); + + // @ts-ignore Mock pushRemoteFeed() is private + expect(call.pushRemoteFeed).not.toHaveBeenCalled(); + }); + + it("correctly pushes", async () => { + // @ts-ignore Mock pushRemoteFeed() is private + jest.spyOn(call, "pushRemoteFeed"); + + await call.placeVoiceCall(); + await call.onAnswerReceived(makeMockEvent("@test:foo", { + version: 1, + call_id: call.callId, + party_id: 'the_correct_party_id', + answer: { + sdp: DUMMY_SDP, + }, + })); + + const stream = new MockMediaStream("stream_ev", [new MockMediaStreamTrack("track_ev", "audio")]); + (call.peerConn as unknown as MockRTCPeerConnection).onTrackListener!( + { streams: [stream], track: stream.getAudioTracks()[0] } as unknown as RTCTrackEvent, + ); + + // @ts-ignore Mock pushRemoteFeed() is private + expect(call.pushRemoteFeed).toHaveBeenCalledWith(stream); + // @ts-ignore Mock pushRemoteFeed() is private + expect(call.removeTrackListeners.has(stream)).toBe(true); + }); + }); + + describe("onHangupReceived()", () => { + it("ends call on onHangupReceived() if state is ringing", async () => { + expect(call.callHasEnded()).toBe(false); + + call.state = CallState.Ringing; + call.onHangupReceived({} as MCallHangupReject); + + expect(call.callHasEnded()).toBe(true); + }); + + it("ends call on onHangupReceived() if party id matches", async () => { + expect(call.callHasEnded()).toBe(false); + + await call.initWithInvite({ + getContent: jest.fn().mockReturnValue({ + version: "1", + call_id: "call_id", + party_id: "remote_party_id", + lifetime: CALL_LIFETIME, + offer: { + sdp: DUMMY_SDP, + }, + }), + getSender: () => "@test:foo", + } as unknown as MatrixEvent); + call.onHangupReceived({ version: "1", party_id: "remote_party_id" } as MCallHangupReject); + + expect(call.callHasEnded()).toBe(true); + }); + }); + + it.each( + Object.values(CallState), + )("ends call on onRejectReceived() if in correct state (state=%s)", async (state: CallState) => { + expect(call.callHasEnded()).toBe(false); + + call.state = state; + call.onRejectReceived({} as MCallHangupReject); + + expect(call.callHasEnded()).toBe( + [CallState.InviteSent, CallState.Ringing, CallState.Ended].includes(state), + ); + }); + + it("terminates call when answered elsewhere", async () => { + await call.placeVoiceCall(); + + expect(call.callHasEnded()).toBe(false); + + call.onAnsweredElsewhere({} as MCallAnswer); + + expect(call.callHasEnded()).toBe(true); + }); + + it("throws when there is no error listener", async () => { + call.off(CallEvent.Error, errorListener); + + expect(call.placeVoiceCall()).rejects.toThrow(); + }); + + describe("hasPeerConnection()", () => { + it("hasPeerConnection() returns false if there is no peer connection", () => { + expect(call.hasPeerConnection).toBe(false); + }); + + it("hasPeerConnection() returns true if there is a peer connection", async () => { + await call.placeVoiceCall(); + expect(call.hasPeerConnection).toBe(true); + }); + }); }); diff --git a/spec/unit/webrtc/callEventHandler.spec.ts b/spec/unit/webrtc/callEventHandler.spec.ts index c13969c7ec2..b7687cdc2fc 100644 --- a/spec/unit/webrtc/callEventHandler.spec.ts +++ b/spec/unit/webrtc/callEventHandler.spec.ts @@ -20,22 +20,117 @@ import { EventTimeline, EventTimelineSet, EventType, + GroupCallIntent, + GroupCallType, IRoomTimelineData, + MatrixCall, MatrixEvent, Room, RoomEvent, + RoomMember, } from "../../../src"; +import { MatrixClient } from "../../../src/client"; import { CallEventHandler, CallEventHandlerEvent } from "../../../src/webrtc/callEventHandler"; +import { GroupCallEventHandler } from "../../../src/webrtc/groupCallEventHandler"; import { SyncState } from "../../../src/sync"; +import { installWebRTCMocks, MockRTCPeerConnection } from "../../test-utils/webrtc"; +import { sleep } from "../../../src/utils"; + +describe("CallEventHandler", () => { + let client: MatrixClient; + beforeEach(() => { + installWebRTCMocks(); + + client = new TestClient("@alice:foo", "somedevice", "token", undefined, {}).client; + client.callEventHandler = new CallEventHandler(client); + client.callEventHandler.start(); + client.groupCallEventHandler = new GroupCallEventHandler(client); + client.groupCallEventHandler.start(); + client.sendStateEvent = jest.fn().mockResolvedValue({}); + }); + + afterEach(() => { + client.callEventHandler!.stop(); + client.groupCallEventHandler!.stop(); + }); + + const sync = async () => { + client.getSyncState = jest.fn().mockReturnValue(SyncState.Syncing); + client.emit(ClientEvent.Sync, SyncState.Syncing, SyncState.Prepared); + + // We can't await the event processing + await sleep(10); + }; + + it("should enforce inbound toDevice message ordering", async () => { + const callEventHandler = client.callEventHandler!; + const event1 = new MatrixEvent({ + type: EventType.CallInvite, + content: { + call_id: "123", + seq: 0, + }, + }); + callEventHandler["onToDeviceEvent"](event1); + + expect(callEventHandler.callEventBuffer.length).toBe(1); + expect(callEventHandler.callEventBuffer[0]).toBe(event1); + + const event2 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 1, + }, + }); + callEventHandler["onToDeviceEvent"](event2); + + expect(callEventHandler.callEventBuffer.length).toBe(2); + expect(callEventHandler.callEventBuffer[1]).toBe(event2); + + const event3 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 3, + }, + }); + callEventHandler["onToDeviceEvent"](event3); + + expect(callEventHandler.callEventBuffer.length).toBe(2); + expect(callEventHandler.nextSeqByCall.get("123")).toBe(2); + expect(callEventHandler.toDeviceEventBuffers.get("123")?.length).toBe(1); + + const event4 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 4, + }, + }); + callEventHandler["onToDeviceEvent"](event4); + + expect(callEventHandler.callEventBuffer.length).toBe(2); + expect(callEventHandler.nextSeqByCall.get("123")).toBe(2); + expect(callEventHandler.toDeviceEventBuffers.get("123")?.length).toBe(2); + + const event5 = new MatrixEvent({ + type: EventType.CallCandidates, + content: { + call_id: "123", + seq: 2, + }, + }); + callEventHandler["onToDeviceEvent"](event5); + + expect(callEventHandler.callEventBuffer.length).toBe(5); + expect(callEventHandler.nextSeqByCall.get("123")).toBe(5); + expect(callEventHandler.toDeviceEventBuffers.get("123")?.length).toBe(0); + }); -describe("callEventHandler", () => { it("should ignore a call if invite & hangup come within a single sync", () => { - const testClient = new TestClient(); - const client = testClient.client; const room = new Room("!room:id", client, "@user:id"); const timelineData: IRoomTimelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; - client.callEventHandler = new CallEventHandler(client); - client.callEventHandler.start(); // Fire off call invite then hangup within a single sync const callInvite = new MatrixEvent({ @@ -62,4 +157,117 @@ describe("callEventHandler", () => { expect(incomingCallEmitted).not.toHaveBeenCalled(); }); + + it("should ignore non-call events", async () => { + // @ts-ignore Mock handleCallEvent is private + jest.spyOn(client.callEventHandler, "handleCallEvent"); + jest.spyOn(client, "checkTurnServers").mockReturnValue(Promise.resolve(true)); + + const room = new Room("!room:id", client, "@user:id"); + const timelineData: IRoomTimelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; + + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.RoomMessage, + room_id: "!room:id", + content: { + text: "hello", + + }, + }), room, false, false, timelineData); + await sync(); + + // @ts-ignore Mock handleCallEvent is private + expect(client.callEventHandler.handleCallEvent).not.toHaveBeenCalled(); + }); + + describe("handleCallEvent()", () => { + const incomingCallListener = jest.fn(); + let timelineData: IRoomTimelineData; + let room: Room; + + beforeEach(() => { + room = new Room("!room:id", client, client.getUserId()!); + timelineData = { timeline: new EventTimeline(new EventTimelineSet(room, {})) }; + + jest.spyOn(client, "checkTurnServers").mockReturnValue(Promise.resolve(true)); + jest.spyOn(client, "getRoom").mockReturnValue(room); + jest.spyOn(room, "getMember").mockReturnValue({ user_id: client.getUserId() } as unknown as RoomMember); + + client.on(CallEventHandlerEvent.Incoming, incomingCallListener); + }); + + afterEach(() => { + MockRTCPeerConnection.resetInstances(); + jest.resetAllMocks(); + }); + + it("should create a call when receiving an invite", async () => { + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.CallInvite, + room_id: "!room:id", + content: { + call_id: "123", + }, + }), room, false, false, timelineData); + await sync(); + + expect(incomingCallListener).toHaveBeenCalled(); + }); + + it("should handle group call event", async () => { + let call: MatrixCall; + const groupCall = await client.createGroupCall( + room.roomId, + GroupCallType.Voice, + false, + GroupCallIntent.Ring, + ); + const SESSION_ID = "sender_session_id"; + const GROUP_CALL_ID = "group_call_id"; + const DEVICE_ID = "device_id"; + + incomingCallListener.mockImplementation((c) => call = c); + jest.spyOn(client.groupCallEventHandler!, "getGroupCallById").mockReturnValue(groupCall); + // @ts-ignore Mock onIncomingCall is private + jest.spyOn(groupCall, "onIncomingCall"); + + await groupCall.enter(); + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.CallInvite, + room_id: "!room:id", + content: { + call_id: "123", + conf_id: GROUP_CALL_ID, + device_id: DEVICE_ID, + sender_session_id: SESSION_ID, + dest_session_id: client.getSessionId(), + }, + }), room, false, false, timelineData); + await sync(); + + expect(incomingCallListener).toHaveBeenCalled(); + expect(call!.groupCallId).toBe(GROUP_CALL_ID); + // @ts-ignore Mock opponentDeviceId is private + expect(call.opponentDeviceId).toBe(DEVICE_ID); + expect(call!.getOpponentSessionId()).toBe(SESSION_ID); + // @ts-ignore Mock onIncomingCall is private + expect(groupCall.onIncomingCall).toHaveBeenCalledWith(call); + + groupCall.terminate(false); + }); + + it("ignores a call with a different invitee than us", async () => { + client.emit(RoomEvent.Timeline, new MatrixEvent({ + type: EventType.CallInvite, + room_id: "!room:id", + content: { + call_id: "123", + invitee: "@bob:bar", + }, + }), room, false, false, timelineData); + await sync(); + + expect(incomingCallListener).not.toHaveBeenCalled(); + }); + }); }); diff --git a/spec/unit/webrtc/groupCall.spec.ts b/spec/unit/webrtc/groupCall.spec.ts new file mode 100644 index 00000000000..fa84490c146 --- /dev/null +++ b/spec/unit/webrtc/groupCall.spec.ts @@ -0,0 +1,1236 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + EventType, + GroupCallIntent, + GroupCallType, + MatrixCall, + MatrixEvent, + Room, + RoomMember, +} from '../../../src'; +import { GroupCall, GroupCallEvent, GroupCallState } from "../../../src/webrtc/groupCall"; +import { MatrixClient } from "../../../src/client"; +import { + installWebRTCMocks, + MockCallFeed, + MockCallMatrixClient, + MockMediaStream, + MockMediaStreamTrack, + MockRTCPeerConnection, +} from '../../test-utils/webrtc'; +import { SDPStreamMetadataKey, SDPStreamMetadataPurpose } from "../../../src/webrtc/callEventTypes"; +import { sleep } from "../../../src/utils"; +import { CallEventHandlerEvent } from '../../../src/webrtc/callEventHandler'; +import { CallFeed } from '../../../src/webrtc/callFeed'; +import { CallEvent, CallState } from '../../../src/webrtc/call'; +import { flushPromises } from '../../test-utils/flushPromises'; + +const FAKE_ROOM_ID = "!fake:test.dummy"; +const FAKE_CONF_ID = "fakegroupcallid"; + +const FAKE_USER_ID_1 = "@alice:test.dummy"; +const FAKE_DEVICE_ID_1 = "@AAAAAA"; +const FAKE_SESSION_ID_1 = "alice1"; +const FAKE_USER_ID_2 = "@bob:test.dummy"; +const FAKE_DEVICE_ID_2 = "@BBBBBB"; +const FAKE_SESSION_ID_2 = "bob1"; +const FAKE_USER_ID_3 = "@charlie:test.dummy"; +const FAKE_STATE_EVENTS = [ + { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + }), + getStateKey: () => FAKE_USER_ID_1, + getRoomId: () => FAKE_ROOM_ID, + }, + { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + ["m.calls"]: [{ + ["m.call_id"]: FAKE_CONF_ID, + ["m.devices"]: [{ + device_id: FAKE_DEVICE_ID_2, + feeds: [], + }], + }], + }), + getStateKey: () => FAKE_USER_ID_2, + getRoomId: () => FAKE_ROOM_ID, + }, { + getContent: () => ({ + ["m.expires_ts"]: Date.now() + ONE_HOUR, + ["m.calls"]: [{ + ["m.call_id"]: FAKE_CONF_ID, + ["m.devices"]: [{ + device_id: "user3_device", + feeds: [], + }], + }], + }), + getStateKey: () => "user3", + getRoomId: () => FAKE_ROOM_ID, + }, +]; + +const ONE_HOUR = 1000 * 60 * 60; + +const createAndEnterGroupCall = async (cli: MatrixClient, room: Room): Promise => { + const groupCall = new GroupCall( + cli, + room, + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + FAKE_CONF_ID, + ); + + await groupCall.create(); + await groupCall.enter(); + + return groupCall; +}; + +class MockCall { + constructor(public roomId: string, public groupCallId: string) { + } + + public state = CallState.Ringing; + public opponentUserId = FAKE_USER_ID_1; + public callId = "1"; + public localUsermediaFeed = { + setAudioVideoMuted: jest.fn(), + stream: new MockMediaStream("stream"), + }; + public remoteUsermediaFeed?: CallFeed; + public remoteScreensharingFeed?: CallFeed; + + public reject = jest.fn(); + public answerWithCallFeeds = jest.fn(); + public hangup = jest.fn(); + + public sendMetadataUpdate = jest.fn(); + + public on = jest.fn(); + public removeListener = jest.fn(); + + public getOpponentMember(): Partial { + return { + userId: this.opponentUserId, + }; + } + + public typed(): MatrixCall { return this as unknown as MatrixCall; } +} + +describe('Group Call', function() { + beforeEach(function() { + installWebRTCMocks(); + }); + + describe('Basic functionality', function() { + let mockSendState: jest.Mock; + let mockClient: MatrixClient; + let room: Room; + let groupCall: GroupCall; + + beforeEach(function() { + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockSendState = typedMockClient.sendStateEvent; + + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + groupCall = new GroupCall(mockClient, room, GroupCallType.Video, false, GroupCallIntent.Prompt); + }); + + it("does not initialize local call feed, if it already is", async () => { + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + + await groupCall.initLocalCallFeed(); + jest.spyOn(groupCall, "initLocalCallFeed"); + await groupCall.enter(); + + expect(groupCall.initLocalCallFeed).not.toHaveBeenCalled(); + + groupCall.leave(); + }); + + it("stops initializing local call feed when leaving", async () => { + const initPromise = groupCall.initLocalCallFeed(); + groupCall.leave(); + await expect(initPromise).rejects.toBeDefined(); + expect(groupCall.state).toBe(GroupCallState.LocalCallFeedUninitialized); + }); + + it("sends state event to room when creating", async () => { + await groupCall.create(); + + expect(mockSendState).toHaveBeenCalledWith( + FAKE_ROOM_ID, EventType.GroupCallPrefix, expect.objectContaining({ + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + }), + groupCall.groupCallId, + ); + }); + + it("sends member state event to room on enter", async () => { + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + + await groupCall.create(); + + try { + await groupCall.enter(); + + expect(mockSendState).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.GroupCallMemberPrefix, + expect.objectContaining({ + "m.calls": [ + expect.objectContaining({ + "m.call_id": groupCall.groupCallId, + "m.devices": [ + expect.objectContaining({ + device_id: FAKE_DEVICE_ID_1, + }), + ], + }), + ], + }), + FAKE_USER_ID_1, + { keepAlive: false }, + ); + } finally { + groupCall.leave(); + } + }); + + it("sends member state event to room on leave", async () => { + room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + + await groupCall.create(); + await groupCall.enter(); + mockSendState.mockClear(); + + groupCall.leave(); + expect(mockSendState).toHaveBeenCalledWith( + FAKE_ROOM_ID, + EventType.GroupCallMemberPrefix, + expect.objectContaining({ "m.calls": [] }), + FAKE_USER_ID_1, + { keepAlive: true }, // Request should outlive the window + ); + }); + + it("starts with mic unmuted in regular calls", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + expect(groupCall.isMicrophoneMuted()).toEqual(false); + } finally { + groupCall.leave(); + } + }); + + it("disables audio stream when audio is set to muted", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + await groupCall.setMicrophoneMuted(true); + + expect(groupCall.isMicrophoneMuted()).toEqual(true); + } finally { + groupCall.leave(); + } + }); + + it("starts with video unmuted in regular calls", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + expect(groupCall.isLocalVideoMuted()).toEqual(false); + } finally { + groupCall.leave(); + } + }); + + it("disables video stream when video is set to muted", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + await groupCall.setLocalVideoMuted(true); + + expect(groupCall.isLocalVideoMuted()).toEqual(true); + } finally { + groupCall.leave(); + } + }); + + it("retains state of local user media stream when updated", async () => { + try { + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + + const oldStream = groupCall.localCallFeed?.stream as unknown as MockMediaStream; + + // arbitrary values, important part is that they're the same afterwards + await groupCall.setLocalVideoMuted(true); + await groupCall.setMicrophoneMuted(false); + + const newStream = await mockClient.getMediaHandler().getUserMediaStream(true, true); + + groupCall.updateLocalUsermediaStream(newStream); + + expect(groupCall.localCallFeed?.stream).toBe(newStream); + + expect(groupCall.isLocalVideoMuted()).toEqual(true); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + + expect(oldStream.isStopped).toEqual(true); + } finally { + groupCall.leave(); + } + }); + + describe("call feeds changing", () => { + let call: MockCall; + const currentFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("current")); + const newFeed = new MockCallFeed(FAKE_USER_ID_1, new MockMediaStream("new")); + + beforeEach(async () => { + jest.spyOn(currentFeed, "dispose"); + jest.spyOn(newFeed, "measureVolumeActivity"); + + jest.spyOn(groupCall, "emit"); + + call = new MockCall(room.roomId, groupCall.groupCallId); + + await groupCall.create(); + }); + + it("ignores changes, if we can't get user id of opponent", async () => { + const call = new MockCall(room.roomId, groupCall.groupCallId); + jest.spyOn(call, "getOpponentMember").mockReturnValue({ userId: undefined }); + + // @ts-ignore Mock + expect(() => groupCall.onCallFeedsChanged(call)).toThrowError(); + }); + + describe("usermedia feeds", () => { + it("adds new usermedia feed", async () => { + call.remoteUsermediaFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.userMediaFeeds).toStrictEqual([newFeed]); + }); + + it("replaces usermedia feed", async () => { + groupCall.userMediaFeeds = [currentFeed.typed()]; + + call.remoteUsermediaFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.userMediaFeeds).toStrictEqual([newFeed]); + }); + + it("removes usermedia feed", async () => { + groupCall.userMediaFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.userMediaFeeds).toHaveLength(0); + }); + }); + + describe("screenshare feeds", () => { + it("adds new screenshare feed", async () => { + call.remoteScreensharingFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.screenshareFeeds).toStrictEqual([newFeed]); + }); + + it("replaces screenshare feed", async () => { + groupCall.screenshareFeeds = [currentFeed.typed()]; + + call.remoteScreensharingFeed = newFeed.typed(); + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.screenshareFeeds).toStrictEqual([newFeed]); + }); + + it("removes screenshare feed", async () => { + groupCall.screenshareFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.onCallFeedsChanged(call); + + expect(groupCall.screenshareFeeds).toHaveLength(0); + }); + }); + + describe("feed replacing", () => { + it("replaces usermedia feed", async () => { + groupCall.userMediaFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.replaceUserMediaFeed(currentFeed, newFeed); + + const newFeeds = [newFeed]; + + expect(groupCall.userMediaFeeds).toStrictEqual(newFeeds); + expect(currentFeed.dispose).toHaveBeenCalled(); + expect(newFeed.measureVolumeActivity).toHaveBeenCalledWith(true); + expect(groupCall.emit).toHaveBeenCalledWith(GroupCallEvent.UserMediaFeedsChanged, newFeeds); + }); + + it("replaces screenshare feed", async () => { + groupCall.screenshareFeeds = [currentFeed.typed()]; + + // @ts-ignore Mock + groupCall.replaceScreenshareFeed(currentFeed, newFeed); + + const newFeeds = [newFeed]; + + expect(groupCall.screenshareFeeds).toStrictEqual(newFeeds); + expect(currentFeed.dispose).toHaveBeenCalled(); + expect(newFeed.measureVolumeActivity).toHaveBeenCalledWith(true); + expect(groupCall.emit).toHaveBeenCalledWith(GroupCallEvent.ScreenshareFeedsChanged, newFeeds); + }); + }); + }); + + describe("PTT calls", () => { + beforeEach(async () => { + // replace groupcall with a PTT one + groupCall = new GroupCall(mockClient, room, GroupCallType.Video, true, GroupCallIntent.Prompt); + + await groupCall.create(); + + await groupCall.initLocalCallFeed(); + }); + + afterEach(() => { + jest.useRealTimers(); + + groupCall.leave(); + }); + + it("starts with mic muted in PTT calls", async () => { + expect(groupCall.isMicrophoneMuted()).toEqual(true); + }); + + it("re-mutes microphone after transmit timeout in PTT mode", async () => { + jest.useFakeTimers(); + + await groupCall.setMicrophoneMuted(false); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + + jest.advanceTimersByTime(groupCall.pttMaxTransmitTime + 100); + + expect(groupCall.isMicrophoneMuted()).toEqual(true); + }); + + it("timer is cleared when mic muted again in PTT mode", async () => { + jest.useFakeTimers(); + + await groupCall.setMicrophoneMuted(false); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + + // 'talk' for half the allowed time + jest.advanceTimersByTime(groupCall.pttMaxTransmitTime / 2); + + await groupCall.setMicrophoneMuted(true); + await groupCall.setMicrophoneMuted(false); + + // we should still be unmuted after almost the full timeout duration + // if not, the timer for the original talking session must have fired + jest.advanceTimersByTime(groupCall.pttMaxTransmitTime - 100); + + expect(groupCall.isMicrophoneMuted()).toEqual(false); + }); + + it("sends metadata updates before unmuting in PTT mode", async () => { + const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); + groupCall.calls.push(mockCall as unknown as MatrixCall); + + let metadataUpdateResolve: () => void; + const metadataUpdatePromise = new Promise(resolve => { + metadataUpdateResolve = resolve; + }); + mockCall.sendMetadataUpdate = jest.fn().mockReturnValue(metadataUpdatePromise); + + const mutePromise = groupCall.setMicrophoneMuted(false); + // we should still be muted at this point because the metadata update hasn't sent + expect(groupCall.isMicrophoneMuted()).toEqual(true); + expect(mockCall.localUsermediaFeed.setAudioVideoMuted).not.toHaveBeenCalled(); + metadataUpdateResolve!(); + + await mutePromise; + + expect(mockCall.localUsermediaFeed.setAudioVideoMuted).toHaveBeenCalled(); + expect(groupCall.isMicrophoneMuted()).toEqual(false); + }); + + it("sends metadata updates after muting in PTT mode", async () => { + const mockCall = new MockCall(FAKE_ROOM_ID, groupCall.groupCallId); + groupCall.calls.push(mockCall as unknown as MatrixCall); + + // the call starts muted, so unmute to get in the right state to test + await groupCall.setMicrophoneMuted(false); + mockCall.localUsermediaFeed.setAudioVideoMuted.mockReset(); + + let metadataUpdateResolve: () => void; + const metadataUpdatePromise = new Promise(resolve => { + metadataUpdateResolve = resolve; + }); + mockCall.sendMetadataUpdate = jest.fn().mockReturnValue(metadataUpdatePromise); + + const mutePromise = groupCall.setMicrophoneMuted(true); + // we should be muted at this point, before the metadata update has been sent + expect(groupCall.isMicrophoneMuted()).toEqual(true); + expect(mockCall.localUsermediaFeed.setAudioVideoMuted).toHaveBeenCalled(); + metadataUpdateResolve!(); + + await mutePromise; + + expect(groupCall.isMicrophoneMuted()).toEqual(true); + }); + }); + }); + + describe('Placing calls', function() { + let groupCall1: GroupCall; + let groupCall2: GroupCall; + let client1: MockCallMatrixClient; + let client2: MockCallMatrixClient; + + beforeEach(function() { + MockRTCPeerConnection.resetInstances(); + + client1 = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + + client2 = new MockCallMatrixClient( + FAKE_USER_ID_2, FAKE_DEVICE_ID_2, FAKE_SESSION_ID_2, + ); + + // Inject the state events directly into each client when sent + const fakeSendStateEvents = ( + roomId: string, eventType: EventType, content: any, statekey: string, + ) => { + if (eventType === EventType.GroupCallMemberPrefix) { + const fakeEvent = { + getContent: () => content, + getRoomId: () => FAKE_ROOM_ID, + getStateKey: () => statekey, + } as unknown as MatrixEvent; + + let subMap = client1Room.currentState.events.get(eventType); + if (!subMap) { + subMap = new Map(); + client1Room.currentState.events.set(eventType, subMap); + client2Room.currentState.events.set(eventType, subMap); + } + // since we cheat & use the same maps for each, we can + // just add it once. + subMap.set(statekey, fakeEvent); + + groupCall1.onMemberStateChanged(fakeEvent); + groupCall2.onMemberStateChanged(fakeEvent); + } + return Promise.resolve({ "event_id": "foo" }); + }; + + client1.sendStateEvent.mockImplementation(fakeSendStateEvents); + client2.sendStateEvent.mockImplementation(fakeSendStateEvents); + + const client1Room = new Room(FAKE_ROOM_ID, client1.typed(), FAKE_USER_ID_1); + + const client2Room = new Room(FAKE_ROOM_ID, client2.typed(), FAKE_USER_ID_2); + + groupCall1 = new GroupCall( + client1.typed(), client1Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, + ); + + groupCall2 = new GroupCall( + client2.typed(), client2Room, GroupCallType.Video, false, GroupCallIntent.Prompt, FAKE_CONF_ID, + ); + + client1Room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + client1Room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + } as unknown as RoomMember; + + client2Room.currentState.members[FAKE_USER_ID_1] = { + userId: FAKE_USER_ID_1, + } as unknown as RoomMember; + client2Room.currentState.members[FAKE_USER_ID_2] = { + userId: FAKE_USER_ID_2, + } as unknown as RoomMember; + }); + + afterEach(function() { + groupCall1.leave(); + groupCall2.leave(); + jest.useRealTimers(); + + MockRTCPeerConnection.resetInstances(); + }); + + it("Places a call to a peer", async function() { + await groupCall1.create(); + + try { + const toDeviceProm = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + await Promise.all([groupCall1.enter(), groupCall2.enter()]); + + MockRTCPeerConnection.triggerAllNegotiations(); + + await toDeviceProm; + + expect(client1.sendToDevice.mock.calls[0][0]).toBe("m.call.invite"); + + const toDeviceCallContent = client1.sendToDevice.mock.calls[0][1]; + expect(Object.keys(toDeviceCallContent).length).toBe(1); + expect(Object.keys(toDeviceCallContent)[0]).toBe(FAKE_USER_ID_2); + + const toDeviceBobDevices = toDeviceCallContent[FAKE_USER_ID_2]; + expect(Object.keys(toDeviceBobDevices).length).toBe(1); + expect(Object.keys(toDeviceBobDevices)[0]).toBe(FAKE_DEVICE_ID_2); + + const bobDeviceMessage = toDeviceBobDevices[FAKE_DEVICE_ID_2]; + expect(bobDeviceMessage.conf_id).toBe(FAKE_CONF_ID); + } finally { + await Promise.all([groupCall1.leave(), groupCall2.leave()]); + } + }); + + it("Retries calls", async function() { + jest.useFakeTimers(); + await groupCall1.create(); + + try { + const toDeviceProm = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + await Promise.all([groupCall1.enter(), groupCall2.enter()]); + + MockRTCPeerConnection.triggerAllNegotiations(); + + await toDeviceProm; + + expect(client1.sendToDevice).toHaveBeenCalled(); + + const oldCall = groupCall1.getCallByUserId(client2.userId); + oldCall!.emit(CallEvent.Hangup, oldCall!); + + client1.sendToDevice.mockClear(); + + const toDeviceProm2 = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + jest.advanceTimersByTime(groupCall1.retryCallInterval + 500); + + // when we placed the call, we could await on enter which waited for the call to + // be made. We don't have that luxury now, so first have to wait for the call + // to even be created... + let newCall: MatrixCall | undefined; + while ( + (newCall = groupCall1.getCallByUserId(client2.userId)) === undefined || + newCall.peerConn === undefined || + newCall.callId == oldCall!.callId + ) { + await flushPromises(); + } + const mockPc = newCall.peerConn as unknown as MockRTCPeerConnection; + + // ...then wait for it to be ready to negotiate + await mockPc.readyToNegotiate; + + MockRTCPeerConnection.triggerAllNegotiations(); + + // ...and then finally we can wait for the invite to be sent + await toDeviceProm2; + + expect(client1.sendToDevice).toHaveBeenCalledWith(EventType.CallInvite, expect.objectContaining({})); + } finally { + await Promise.all([groupCall1.leave(), groupCall2.leave()]); + } + }); + + it("Updates call mute status correctly on call state change", async function() { + await groupCall1.create(); + + try { + const toDeviceProm = new Promise(resolve => { + client1.sendToDevice.mockImplementation(() => { + resolve(); + return Promise.resolve({}); + }); + }); + + await Promise.all([groupCall1.enter(), groupCall2.enter()]); + + MockRTCPeerConnection.triggerAllNegotiations(); + + await toDeviceProm; + + groupCall1.setMicrophoneMuted(false); + groupCall1.setLocalVideoMuted(false); + + const call = groupCall1.getCallByUserId(client2.userId)!; + call.isMicrophoneMuted = jest.fn().mockReturnValue(true); + call.setMicrophoneMuted = jest.fn(); + call.isLocalVideoMuted = jest.fn().mockReturnValue(true); + call.setLocalVideoMuted = jest.fn(); + + call.emit(CallEvent.State, CallState.Connected); + + expect(call.setMicrophoneMuted).toHaveBeenCalledWith(false); + expect(call.setLocalVideoMuted).toHaveBeenCalledWith(false); + } finally { + await Promise.all([groupCall1.leave(), groupCall2.leave()]); + } + }); + }); + + describe("muting", () => { + let mockClient: MatrixClient; + let room: Room; + + beforeEach(() => { + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { + return type === EventType.GroupCallMemberPrefix + ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS + : { getContent: () => ([]) }; + }); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + }); + + describe("local muting", () => { + it("should mute local audio when calling setMicrophoneMuted()", async () => { + const groupCall = await createAndEnterGroupCall(mockClient, room); + + groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); + const setAVMutedArray = groupCall.calls.map(call => { + call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); + return call.localUsermediaFeed!.setAudioVideoMuted; + }); + const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { + acc.push(...call.localUsermediaStream!.getAudioTracks()); + return acc; + }, []); + const sendMetadataUpdateArray = groupCall.calls.map(call => { + call.sendMetadataUpdate = jest.fn(); + return call.sendMetadataUpdate; + }); + + await groupCall.setMicrophoneMuted(true); + + groupCall.localCallFeed!.stream.getAudioTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(true, null); + setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(true, null)); + tracksArray.forEach(track => expect(track.enabled).toBe(false)); + sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + + it("should mute local video when calling setLocalVideoMuted()", async () => { + const groupCall = await createAndEnterGroupCall(mockClient, room); + + groupCall.localCallFeed!.setAudioVideoMuted = jest.fn(); + const setAVMutedArray = groupCall.calls.map(call => { + call.localUsermediaFeed!.setAudioVideoMuted = jest.fn(); + call.localUsermediaFeed!.isVideoMuted = jest.fn().mockReturnValue(true); + return call.localUsermediaFeed!.setAudioVideoMuted; + }); + const tracksArray = groupCall.calls.reduce((acc: MediaStreamTrack[], call: MatrixCall) => { + acc.push(...call.localUsermediaStream!.getVideoTracks()); + return acc; + }, []); + const sendMetadataUpdateArray = groupCall.calls.map(call => { + call.sendMetadataUpdate = jest.fn(); + return call.sendMetadataUpdate; + }); + + await groupCall.setLocalVideoMuted(true); + + groupCall.localCallFeed!.stream.getVideoTracks().forEach(track => expect(track.enabled).toBe(false)); + expect(groupCall.localCallFeed!.setAudioVideoMuted).toHaveBeenCalledWith(null, true); + setAVMutedArray.forEach(f => expect(f).toHaveBeenCalledWith(null, true)); + tracksArray.forEach(track => expect(track.enabled).toBe(false)); + sendMetadataUpdateArray.forEach(f => expect(f).toHaveBeenCalled()); + + groupCall.terminate(); + }); + }); + + describe("remote muting", () => { + const getMetadataEvent = (audio: boolean, video: boolean): MatrixEvent => ({ + getContent: () => ({ + [SDPStreamMetadataKey]: { + stream: { + purpose: SDPStreamMetadataPurpose.Usermedia, + audio_muted: audio, + video_muted: video, + }, + }, + }), + } as MatrixEvent); + + it("should mute remote feed's audio after receiving metadata with video audio", async () => { + const metadataEvent = getMetadataEvent(true, false); + const groupCall = await createAndEnterGroupCall(mockClient, room); + + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("audio_track", "audio"), + new MockMediaStreamTrack("video_track", "video"), + ])); + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); + expect(feed!.isAudioMuted()).toBe(true); + expect(feed!.isVideoMuted()).toBe(false); + + groupCall.terminate(); + }); + + it("should mute remote feed's video after receiving metadata with video muted", async () => { + const metadataEvent = getMetadataEvent(false, true); + const groupCall = await createAndEnterGroupCall(mockClient, room); + + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("stream", [ + new MockMediaStreamTrack("audio_track", "audio"), + new MockMediaStreamTrack("video_track", "video"), + ])); + call.onSDPStreamMetadataChangedReceived(metadataEvent); + + const feed = groupCall.getUserMediaFeedByUserId(call.invitee!); + expect(feed!.isAudioMuted()).toBe(false); + expect(feed!.isVideoMuted()).toBe(true); + + groupCall.terminate(); + }); + }); + }); + + describe("incoming calls", () => { + let mockClient: MatrixClient; + let room: Room; + let groupCall: GroupCall; + + beforeEach(async () => { + // we are bob here because we're testing incoming calls, and since alice's user id + // is lexicographically before Bob's, the spec requires that she calls Bob. + const typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_2, FAKE_DEVICE_ID_2, FAKE_SESSION_ID_2, + ); + mockClient = typedMockClient as unknown as MatrixClient; + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_2); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + + groupCall = await createAndEnterGroupCall(mockClient, room); + }); + + afterEach(() => { + groupCall.leave(); + }); + + it("ignores incoming calls for other rooms", async () => { + const mockCall = new MockCall("!someotherroom.fake.dummy", groupCall.groupCallId); + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).not.toHaveBeenCalled(); + expect(mockCall.answerWithCallFeeds).not.toHaveBeenCalled(); + }); + + it("rejects incoming calls for the wrong group call", async () => { + const mockCall = new MockCall(room.roomId, "not " + groupCall.groupCallId); + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).toHaveBeenCalled(); + }); + + it("ignores incoming calls not in the ringing state", async () => { + const mockCall = new MockCall(room.roomId, groupCall.groupCallId); + mockCall.state = CallState.Connected; + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).not.toHaveBeenCalled(); + expect(mockCall.answerWithCallFeeds).not.toHaveBeenCalled(); + }); + + it("answers calls for the right room & group call ID", async () => { + const mockCall = new MockCall(room.roomId, groupCall.groupCallId); + + mockClient.emit(CallEventHandlerEvent.Incoming, mockCall as unknown as MatrixCall); + + expect(mockCall.reject).not.toHaveBeenCalled(); + expect(mockCall.answerWithCallFeeds).toHaveBeenCalled(); + expect(groupCall.calls).toEqual([mockCall]); + }); + + it("replaces calls if it already has one with the same user", async () => { + const oldMockCall = new MockCall(room.roomId, groupCall.groupCallId); + const newMockCall = new MockCall(room.roomId, groupCall.groupCallId); + newMockCall.callId = "not " + oldMockCall.callId; + + mockClient.emit(CallEventHandlerEvent.Incoming, oldMockCall as unknown as MatrixCall); + mockClient.emit(CallEventHandlerEvent.Incoming, newMockCall as unknown as MatrixCall); + + expect(oldMockCall.hangup).toHaveBeenCalled(); + expect(newMockCall.answerWithCallFeeds).toHaveBeenCalled(); + expect(groupCall.calls).toEqual([newMockCall]); + }); + + it("starts to process incoming calls when we've entered", async () => { + // First we leave the call since we have already entered + groupCall.leave(); + + const call = new MockCall(room.roomId, groupCall.groupCallId); + mockClient.callEventHandler!.calls = new Map([ + [call.callId, call.typed()], + ]); + await groupCall.enter(); + + expect(call.answerWithCallFeeds).toHaveBeenCalled(); + }); + }); + + describe("screensharing", () => { + let typedMockClient: MockCallMatrixClient; + let mockClient: MatrixClient; + let room: Room; + let groupCall: GroupCall; + + beforeEach(async () => { + typedMockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + mockClient = typedMockClient.typed(); + + room = new Room(FAKE_ROOM_ID, mockClient, FAKE_USER_ID_1); + room.getMember = jest.fn().mockImplementation((userId) => ({ userId })); + room.currentState.getStateEvents = jest.fn().mockImplementation((type: EventType, userId: string) => { + return type === EventType.GroupCallMemberPrefix + ? FAKE_STATE_EVENTS.find(e => e.getStateKey() === userId) || FAKE_STATE_EVENTS + : { getContent: () => ([]) }; + }); + + groupCall = await createAndEnterGroupCall(mockClient, room); + }); + + it("sending screensharing stream", async () => { + const onNegotiationNeededArray = groupCall.calls.map(call => { + // @ts-ignore Mock + call.gotLocalOffer = jest.fn(); + // @ts-ignore Mock + return call.gotLocalOffer; + }); + + let enabledResult; + enabledResult = await groupCall.setScreensharingEnabled(true); + expect(enabledResult).toEqual(true); + expect(typedMockClient.mediaHandler.getScreensharingStream).toHaveBeenCalled(); + MockRTCPeerConnection.triggerAllNegotiations(); + + expect(groupCall.screenshareFeeds).toHaveLength(1); + groupCall.calls.forEach(c => { + expect(c.getLocalFeeds().find(f => f.purpose === SDPStreamMetadataPurpose.Screenshare)).toBeDefined(); + }); + onNegotiationNeededArray.forEach(f => expect(f).toHaveBeenCalled()); + + // Enabling it again should do nothing + typedMockClient.mediaHandler.getScreensharingStream.mockClear(); + enabledResult = await groupCall.setScreensharingEnabled(true); + expect(enabledResult).toEqual(true); + expect(typedMockClient.mediaHandler.getScreensharingStream).not.toHaveBeenCalled(); + + // Should now be able to disable it + enabledResult = await groupCall.setScreensharingEnabled(false); + expect(enabledResult).toEqual(false); + expect(groupCall.screenshareFeeds).toHaveLength(0); + + groupCall.terminate(); + }); + + it("receiving screensharing stream", async () => { + // It takes a bit of time for the calls to get created + await sleep(10); + + const call = groupCall.calls[0]; + call.getOpponentMember = () => ({ userId: call.invitee }) as RoomMember; + call.onNegotiateReceived({ + getContent: () => ({ + [SDPStreamMetadataKey]: { + "screensharing_stream": { + purpose: SDPStreamMetadataPurpose.Screenshare, + }, + }, + description: { + type: "offer", + sdp: "...", + }, + }), + } as MatrixEvent); + // @ts-ignore Mock + call.pushRemoteFeed(new MockMediaStream("screensharing_stream", [ + new MockMediaStreamTrack("video_track", "video"), + ])); + + expect(groupCall.screenshareFeeds).toHaveLength(1); + expect(groupCall.getScreenshareFeedByUserId(call.invitee!)).toBeDefined(); + + groupCall.terminate(); + }); + + it("cleans up screensharing when terminating", async () => { + // @ts-ignore Mock + jest.spyOn(groupCall, "removeScreenshareFeed"); + jest.spyOn(mockClient.getMediaHandler(), "stopScreensharingStream"); + + await groupCall.setScreensharingEnabled(true); + + const screensharingFeed = groupCall.localScreenshareFeed; + + groupCall.terminate(); + + expect(mockClient.getMediaHandler()!.stopScreensharingStream).toHaveBeenCalledWith( + screensharingFeed!.stream, + ); + // @ts-ignore Mock + expect(groupCall.removeScreenshareFeed).toHaveBeenCalledWith(screensharingFeed); + expect(groupCall.localScreenshareFeed).toBeUndefined(); + }); + }); + + describe("active speaker events", () => { + let room: Room; + let groupCall: GroupCall; + let mediaFeed1: CallFeed; + let mediaFeed2: CallFeed; + let onActiveSpeakerEvent: jest.Mock; + + beforeEach(async () => { + jest.useFakeTimers(); + + const mockClient = new MockCallMatrixClient( + FAKE_USER_ID_1, FAKE_DEVICE_ID_1, FAKE_SESSION_ID_1, + ); + + room = new Room(FAKE_ROOM_ID, mockClient.typed(), FAKE_USER_ID_1); + groupCall = await createAndEnterGroupCall(mockClient.typed(), room); + + mediaFeed1 = new CallFeed({ + client: mockClient.typed(), + roomId: FAKE_ROOM_ID, + userId: FAKE_USER_ID_2, + stream: (new MockMediaStream("foo", [])).typed(), + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: true, + }); + groupCall.userMediaFeeds.push(mediaFeed1); + + mediaFeed2 = new CallFeed({ + client: mockClient.typed(), + roomId: FAKE_ROOM_ID, + userId: FAKE_USER_ID_3, + stream: (new MockMediaStream("foo", [])).typed(), + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: false, + videoMuted: true, + }); + groupCall.userMediaFeeds.push(mediaFeed2); + + onActiveSpeakerEvent = jest.fn(); + groupCall.on(GroupCallEvent.ActiveSpeakerChanged, onActiveSpeakerEvent); + }); + + afterEach(() => { + groupCall.off(GroupCallEvent.ActiveSpeakerChanged, onActiveSpeakerEvent); + + jest.useRealTimers(); + }); + + it("fires active speaker events when a user is speaking", async () => { + mediaFeed1.speakingVolumeSamples = [100, 100]; + mediaFeed2.speakingVolumeSamples = [0, 0]; + + jest.runOnlyPendingTimers(); + expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_2); + expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_2); + + mediaFeed1.speakingVolumeSamples = [0, 0]; + mediaFeed2.speakingVolumeSamples = [100, 100]; + + jest.runOnlyPendingTimers(); + expect(groupCall.activeSpeaker).toEqual(FAKE_USER_ID_3); + expect(onActiveSpeakerEvent).toHaveBeenCalledWith(FAKE_USER_ID_3); + }); + }); + + describe("creating group calls", () => { + let client: MatrixClient; + + beforeEach(() => { + client = new MatrixClient({ baseUrl: "base_url" }); + + jest.spyOn(client, "sendStateEvent").mockResolvedValue({} as any); + }); + + afterEach(() => { + client.stopClient(); + }); + + it("throws when there already is a call", async () => { + jest.spyOn(client, "getRoom").mockReturnValue(new Room("room_id", client, "my_user_id")); + + await client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + ); + + await expect(client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + )).rejects.toThrow("room_id already has an existing group call"); + }); + + it("throws if the room doesn't exist", async () => { + await expect(client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + )).rejects.toThrow("Cannot find room room_id"); + }); + + describe("correctly passes parameters", () => { + beforeEach(() => { + jest.spyOn(client, "getRoom").mockReturnValue(new Room("room_id", client, "my_user_id")); + }); + + it("correctly passes voice ptt room call", async () => { + const groupCall = await client.createGroupCall( + "room_id", + GroupCallType.Voice, + true, + GroupCallIntent.Room, + ); + + expect(groupCall.type).toBe(GroupCallType.Voice); + expect(groupCall.isPtt).toBe(true); + expect(groupCall.intent).toBe(GroupCallIntent.Room); + }); + + it("correctly passes voice ringing call", async () => { + const groupCall = await client.createGroupCall( + "room_id", + GroupCallType.Voice, + false, + GroupCallIntent.Ring, + ); + + expect(groupCall.type).toBe(GroupCallType.Voice); + expect(groupCall.isPtt).toBe(false); + expect(groupCall.intent).toBe(GroupCallIntent.Ring); + }); + + it("correctly passes video prompt call", async () => { + const groupCall = await client.createGroupCall( + "room_id", + GroupCallType.Video, + false, + GroupCallIntent.Prompt, + ); + + expect(groupCall.type).toBe(GroupCallType.Video); + expect(groupCall.isPtt).toBe(false); + expect(groupCall.intent).toBe(GroupCallIntent.Prompt); + }); + }); + }); +}); diff --git a/spec/unit/webrtc/groupCallEventHandler.spec.ts b/spec/unit/webrtc/groupCallEventHandler.spec.ts new file mode 100644 index 00000000000..6712b0f09f0 --- /dev/null +++ b/spec/unit/webrtc/groupCallEventHandler.spec.ts @@ -0,0 +1,316 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { mocked } from "jest-mock"; + +import { + ClientEvent, + GroupCall, + GroupCallIntent, + GroupCallState, + GroupCallType, + IContent, + MatrixEvent, + Room, + RoomState, +} from "../../../src"; +import { SyncState } from "../../../src/sync"; +import { GroupCallTerminationReason } from "../../../src/webrtc/groupCall"; +import { GroupCallEventHandler, GroupCallEventHandlerEvent } from "../../../src/webrtc/groupCallEventHandler"; +import { flushPromises } from "../../test-utils/flushPromises"; +import { + makeMockGroupCallMemberStateEvent, + makeMockGroupCallStateEvent, + MockCallMatrixClient, +} from "../../test-utils/webrtc"; + +const FAKE_USER_ID = "@alice:test.dummy"; +const FAKE_DEVICE_ID = "AAAAAAA"; +const FAKE_SESSION_ID = "session1"; +const FAKE_ROOM_ID = "!roomid:test.dummy"; +const FAKE_GROUP_CALL_ID = "fakegroupcallid"; + +describe('Group Call Event Handler', function() { + let groupCallEventHandler: GroupCallEventHandler; + let mockClient: MockCallMatrixClient; + let mockRoom: Room; + + beforeEach(() => { + mockClient = new MockCallMatrixClient( + FAKE_USER_ID, FAKE_DEVICE_ID, FAKE_SESSION_ID, + ); + groupCallEventHandler = new GroupCallEventHandler(mockClient.typed()); + + mockRoom = { + roomId: FAKE_ROOM_ID, + currentState: { + getStateEvents: jest.fn().mockReturnValue([makeMockGroupCallStateEvent( + FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, + )]), + }, + } as unknown as Room; + + (mockClient as any).getRoom = jest.fn().mockReturnValue(mockRoom); + }); + + describe("reacts to state changes", () => { + it("terminates call", async () => { + await groupCallEventHandler.start(); + mockClient.emitRoomState( + makeMockGroupCallStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID), + { roomId: FAKE_ROOM_ID } as unknown as RoomState, + ); + + const groupCall = groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID)!; + + expect(groupCall.state).toBe(GroupCallState.LocalCallFeedUninitialized); + + mockClient.emitRoomState( + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, { + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + "m.terminated": GroupCallTerminationReason.CallEnded, + }, + ), + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + ); + + expect(groupCall.state).toBe(GroupCallState.Ended); + }); + }); + + it("waits until client starts syncing", async () => { + mockClient.getSyncState.mockReturnValue(null); + let isStarted = false; + (async () => { + await groupCallEventHandler.start(); + isStarted = true; + })(); + + const setSyncState = async (newState: SyncState) => { + const oldState = mockClient.getSyncState(); + mockClient.getSyncState.mockReturnValue(newState); + mockClient.emit(ClientEvent.Sync, newState, oldState, undefined); + await flushPromises(); + }; + + await flushPromises(); + expect(isStarted).toEqual(false); + + await setSyncState(SyncState.Prepared); + expect(isStarted).toEqual(false); + + await setSyncState(SyncState.Syncing); + expect(isStarted).toEqual(true); + }); + + it("finds existing group calls when started", async () => { + const mockClientEmit = mockClient.emit = jest.fn(); + + mockClient.getRooms.mockReturnValue([mockRoom]); + await groupCallEventHandler.start(); + + expect(mockClientEmit).toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + + groupCallEventHandler.stop(); + }); + + it("can wait until a room is ready for group calls", async () => { + await groupCallEventHandler.start(); + + const prom = groupCallEventHandler.waitUntilRoomReadyForGroupCalls(FAKE_ROOM_ID); + let resolved = false; + + (async () => { + await prom; + resolved = true; + })(); + + expect(resolved).toEqual(false); + mockClient.emit(ClientEvent.Room, mockRoom); + + await prom; + expect(resolved).toEqual(true); + + groupCallEventHandler.stop(); + }); + + it("fires events for incoming calls", async () => { + const onIncomingGroupCall = jest.fn(); + mockClient.on(GroupCallEventHandlerEvent.Incoming, onIncomingGroupCall); + await groupCallEventHandler.start(); + + mockClient.emitRoomState( + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, FAKE_GROUP_CALL_ID, + ), + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + ); + + expect(onIncomingGroupCall).toHaveBeenCalledWith(expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + })); + + mockClient.off(GroupCallEventHandlerEvent.Incoming, onIncomingGroupCall); + }); + + it("handles data channel", async () => { + await groupCallEventHandler.start(); + + const dataChannelOptions = { + "maxPacketLifeTime": "life_time", + "maxRetransmits": "retransmits", + "ordered": "ordered", + "protocol": "protocol", + }; + + mockClient.emitRoomState( + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, + FAKE_GROUP_CALL_ID, + { + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + "dataChannelsEnabled": true, + dataChannelOptions, + }, + ), + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + ); + + // @ts-ignore Mock dataChannelsEnabled is private + expect(groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID)?.dataChannelsEnabled).toBe(true); + // @ts-ignore Mock dataChannelOptions is private + expect(groupCallEventHandler.groupCalls.get(FAKE_ROOM_ID)?.dataChannelOptions).toStrictEqual( + dataChannelOptions, + ); + }); + + it("sends member events to group calls", async () => { + await groupCallEventHandler.start(); + + const mockGroupCall = { + onMemberStateChanged: jest.fn(), + }; + + groupCallEventHandler.groupCalls.set(FAKE_ROOM_ID, mockGroupCall as unknown as GroupCall); + + const mockStateEvent = makeMockGroupCallMemberStateEvent(FAKE_ROOM_ID, FAKE_GROUP_CALL_ID); + + mockClient.emitRoomState( + mockStateEvent, + { + roomId: FAKE_ROOM_ID, + } as unknown as RoomState, + ); + + expect(mockGroupCall.onMemberStateChanged).toHaveBeenCalledWith(mockStateEvent); + }); + + describe("ignoring invalid group call state events", () => { + let mockClientEmit: jest.Func; + + beforeEach(() => { + mockClientEmit = mockClient.emit = jest.fn(); + }); + + afterEach(() => { + groupCallEventHandler.stop(); + + jest.clearAllMocks(); + }); + + const setupCallAndStart = async (content?: IContent) => { + mocked(mockRoom.currentState.getStateEvents).mockReturnValue([ + makeMockGroupCallStateEvent( + FAKE_ROOM_ID, + FAKE_GROUP_CALL_ID, + content, + ), + ] as unknown as MatrixEvent); + mockClient.getRooms.mockReturnValue([mockRoom]); + await groupCallEventHandler.start(); + }; + + it("ignores terminated calls", async () => { + await setupCallAndStart({ + "m.type": GroupCallType.Video, + "m.intent": GroupCallIntent.Prompt, + "m.terminated": GroupCallTerminationReason.CallEnded, + }); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + + it("ignores calls with invalid type", async () => { + await setupCallAndStart({ + "m.type": "fake_type", + "m.intent": GroupCallIntent.Prompt, + }); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + + it("ignores calls with invalid intent", async () => { + await setupCallAndStart({ + "m.type": GroupCallType.Video, + "m.intent": "fake_intent", + }); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + + it("ignores calls without a room", async () => { + mockClient.getRoom.mockReturnValue(undefined); + + await setupCallAndStart(); + + expect(mockClientEmit).not.toHaveBeenCalledWith( + GroupCallEventHandlerEvent.Incoming, + expect.objectContaining({ + groupCallId: FAKE_GROUP_CALL_ID, + }), + ); + }); + }); +}); diff --git a/spec/unit/webrtc/mediaHandler.spec.ts b/spec/unit/webrtc/mediaHandler.spec.ts new file mode 100644 index 00000000000..94396845c96 --- /dev/null +++ b/spec/unit/webrtc/mediaHandler.spec.ts @@ -0,0 +1,445 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { GroupCall, MatrixCall, MatrixClient } from "../../../src"; +import { MediaHandler, MediaHandlerEvent } from "../../../src/webrtc/mediaHandler"; +import { MockMediaDeviceInfo, MockMediaDevices, MockMediaStream, MockMediaStreamTrack } from "../../test-utils/webrtc"; + +const FAKE_AUDIO_INPUT_ID = "aaaaaaaa"; +const FAKE_VIDEO_INPUT_ID = "vvvvvvvv"; +const FAKE_DESKTOP_SOURCE_ID = "ddddddd"; + +describe('Media Handler', function() { + let mockMediaDevices: MockMediaDevices; + let mediaHandler: MediaHandler; + let calls: Map; + let groupCalls: Map; + + beforeEach(() => { + mockMediaDevices = new MockMediaDevices(); + + global.navigator = { + mediaDevices: mockMediaDevices.typed(), + } as unknown as Navigator; + + calls = new Map(); + groupCalls = new Map(); + + mediaHandler = new MediaHandler({ + callEventHandler: { + calls, + }, + groupCallEventHandler: { + groupCalls, + }, + } as unknown as MatrixClient); + }); + + it("does not trigger update after restore media settings ", () => { + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + expect(mockMediaDevices.getUserMedia).not.toHaveBeenCalled(); + }); + + it("sets device IDs on restore media settings", async () => { + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + it("sets audio device ID", async () => { + await mediaHandler.setAudioInput(FAKE_AUDIO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, false); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + })); + }); + + it("sets video device ID", async () => { + await mediaHandler.setVideoInput(FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(false, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + it("sets media inputs", async () => { + await mediaHandler.setMediaInputs(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + + await mediaHandler.getUserMediaStream(true, true); + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + audio: expect.objectContaining({ + deviceId: { ideal: FAKE_AUDIO_INPUT_ID }, + }), + video: expect.objectContaining({ + deviceId: { ideal: FAKE_VIDEO_INPUT_ID }, + }), + })); + }); + + describe("updateLocalUsermediaStreams", () => { + let localStreamsChangedHandler: jest.Mock; + + beforeEach(() => { + localStreamsChangedHandler = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, localStreamsChangedHandler); + }); + + afterEach(() => { + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, localStreamsChangedHandler); + }); + + it("does nothing if it has no streams", async () => { + mediaHandler.updateLocalUsermediaStreams(); + expect(mockMediaDevices.getUserMedia).not.toHaveBeenCalled(); + }); + + it("does not emit LocalStreamsChanged if it had no streams", async () => { + await mediaHandler.updateLocalUsermediaStreams(); + + expect(localStreamsChangedHandler).not.toHaveBeenCalled(); + }); + + describe("with existing streams", () => { + let stopTrack: jest.Mock; + let updateLocalUsermediaStream: jest.Mock; + + beforeEach(() => { + stopTrack = jest.fn(); + + mediaHandler.userMediaStreams = [ + { + getTracks: () => [{ + stop: stopTrack, + } as unknown as MediaStreamTrack], + } as unknown as MediaStream, + ]; + + updateLocalUsermediaStream = jest.fn(); + }); + + it("stops existing streams", async () => { + mediaHandler.updateLocalUsermediaStreams(); + expect(stopTrack).toHaveBeenCalled(); + }); + + it("replaces streams on calls", async () => { + calls.set("some_call", { + hasLocalUserMediaAudioTrack: true, + hasLocalUserMediaVideoTrack: true, + callHasEnded: jest.fn().mockReturnValue(false), + updateLocalUsermediaStream, + } as unknown as MatrixCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).toHaveBeenCalled(); + }); + + it("doesn't replace streams on ended calls", async () => { + calls.set("some_call", { + hasLocalUserMediaAudioTrack: true, + hasLocalUserMediaVideoTrack: true, + callHasEnded: jest.fn().mockReturnValue(true), + updateLocalUsermediaStream, + } as unknown as MatrixCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).not.toHaveBeenCalled(); + }); + + it("replaces streams on group calls", async () => { + groupCalls.set("some_group_call", { + localCallFeed: {}, + updateLocalUsermediaStream, + } as unknown as GroupCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).toHaveBeenCalled(); + }); + + it("doesn't replace streams on group calls with no localCallFeed", async () => { + groupCalls.set("some_group_call", { + localCallFeed: null, + updateLocalUsermediaStream, + } as unknown as GroupCall); + + await mediaHandler.updateLocalUsermediaStreams(); + expect(updateLocalUsermediaStream).not.toHaveBeenCalled(); + }); + + it("emits LocalStreamsChanged", async () => { + await mediaHandler.updateLocalUsermediaStreams(); + + expect(localStreamsChangedHandler).toHaveBeenCalled(); + }); + }); + }); + + describe("hasAudioDevice", () => { + it("returns true if the system has audio inputs", async () => { + expect(await mediaHandler.hasAudioDevice()).toEqual(true); + }); + + it("returns false if the system has no audio inputs", async () => { + mockMediaDevices.enumerateDevices.mockReturnValue(Promise.resolve([ + new MockMediaDeviceInfo("videoinput").typed(), + ])); + expect(await mediaHandler.hasAudioDevice()).toEqual(false); + }); + }); + + describe("hasVideoDevice", () => { + it("returns true if the system has video inputs", async () => { + expect(await mediaHandler.hasVideoDevice()).toEqual(true); + }); + + it("returns false if the system has no video inputs", async () => { + mockMediaDevices.enumerateDevices.mockReturnValue(Promise.resolve([ + new MockMediaDeviceInfo("audioinput").typed(), + ])); + expect(await mediaHandler.hasVideoDevice()).toEqual(false); + }); + }); + + describe("getUserMediaStream", () => { + beforeEach(() => { + // replace this with one that returns a new object each time so we can + // tell whether we've ended up with the same stream + mockMediaDevices.getUserMedia.mockImplementation((constraints: MediaStreamConstraints) => { + const stream = new MockMediaStream("local_stream"); + if (constraints.audio) { + const track = new MockMediaStreamTrack("audio_track", "audio"); + track.settings = { deviceId: FAKE_AUDIO_INPUT_ID }; + stream.addTrack(track); + } + if (constraints.video) { + const track = new MockMediaStreamTrack("video_track", "video"); + track.settings = { deviceId: FAKE_VIDEO_INPUT_ID }; + stream.addTrack(track); + } + + return Promise.resolve(stream.typed()); + }); + + mediaHandler.restoreMediaSettings(FAKE_AUDIO_INPUT_ID, FAKE_VIDEO_INPUT_ID); + }); + + it("returns the same stream for reusable streams", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + }); + + it("doesn't re-use stream if reusable is false", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false, false); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("doesn't re-use stream if existing stream lacks audio", async () => { + const stream1 = await mediaHandler.getUserMediaStream(false, true); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("doesn't re-use stream if existing stream lacks video", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, false); + const stream2 = await mediaHandler.getUserMediaStream(false, true) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(false); + }); + + it("strips unwanted audio tracks from re-used stream", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, true); + const stream2 = await mediaHandler.getUserMediaStream(false, true) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + expect(stream2.getAudioTracks().length).toEqual(0); + }); + + it("strips unwanted video tracks from re-used stream", async () => { + const stream1 = await mediaHandler.getUserMediaStream(true, true); + const stream2 = await mediaHandler.getUserMediaStream(true, false) as unknown as MockMediaStream; + + expect(stream2.isCloneOf(stream1)).toEqual(true); + expect(stream2.getVideoTracks().length).toEqual(0); + }); + }); + + describe("getScreensharingStream", () => { + it("gets any screen sharing stream when called with no args", async () => { + const stream = await mediaHandler.getScreensharingStream(); + expect(stream).toBeTruthy(); + expect(stream.getTracks()).toBeTruthy(); + }); + + it("re-uses streams", async () => { + const stream = await mediaHandler.getScreensharingStream(undefined, true); + + expect(mockMediaDevices.getDisplayMedia).toHaveBeenCalled(); + mockMediaDevices.getDisplayMedia.mockClear(); + + const stream2 = await mediaHandler.getScreensharingStream() as unknown as MockMediaStream; + + expect(mockMediaDevices.getDisplayMedia).not.toHaveBeenCalled(); + + expect(stream2.isCloneOf(stream)).toEqual(true); + }); + + it("passes through desktopCapturerSourceId for Electron", async () => { + await mediaHandler.getScreensharingStream({ + desktopCapturerSourceId: FAKE_DESKTOP_SOURCE_ID, + }); + + expect(mockMediaDevices.getUserMedia).toHaveBeenCalledWith(expect.objectContaining({ + video: { + mandatory: expect.objectContaining({ + chromeMediaSource: "desktop", + chromeMediaSourceId: FAKE_DESKTOP_SOURCE_ID, + }), + }, + })); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + await mediaHandler.getScreensharingStream(); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopUserMediaStream", () => { + let stream: MediaStream; + + beforeEach(async () => { + stream = await mediaHandler.getUserMediaStream(true, false); + }); + + it("stops tracks on streams", async () => { + const mockTrack = new MockMediaStreamTrack("audio_track", "audio"); + stream.addTrack(mockTrack.typed()); + + mediaHandler.stopUserMediaStream(stream); + expect(mockTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.userMediaStreams).toContain(stream); + mediaHandler.stopUserMediaStream(stream); + expect(mediaHandler.userMediaStreams).not.toContain(stream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopUserMediaStream(stream); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopUserMediaStream", () => { + let stream: MediaStream; + + beforeEach(async () => { + stream = await mediaHandler.getScreensharingStream(); + }); + + it("stops tracks on streams", async () => { + const mockTrack = new MockMediaStreamTrack("audio_track", "audio"); + stream.addTrack(mockTrack.typed()); + + mediaHandler.stopScreensharingStream(stream); + expect(mockTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.screensharingStreams).toContain(stream); + mediaHandler.stopScreensharingStream(stream); + expect(mediaHandler.screensharingStreams).not.toContain(stream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopScreensharingStream(stream); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); + + describe("stopAllStreams", () => { + let userMediaStream: MediaStream; + let screenSharingStream: MediaStream; + + beforeEach(async () => { + userMediaStream = await mediaHandler.getUserMediaStream(true, false); + screenSharingStream = await mediaHandler.getScreensharingStream(); + }); + + it("stops tracks on streams", async () => { + const mockUserMediaTrack = new MockMediaStreamTrack("audio_track", "audio"); + userMediaStream.addTrack(mockUserMediaTrack.typed()); + + const mockScreenshareTrack = new MockMediaStreamTrack("audio_track", "audio"); + screenSharingStream.addTrack(mockScreenshareTrack.typed()); + + mediaHandler.stopAllStreams(); + + expect(mockUserMediaTrack.stop).toHaveBeenCalled(); + expect(mockScreenshareTrack.stop).toHaveBeenCalled(); + }); + + it("removes stopped streams", async () => { + expect(mediaHandler.userMediaStreams).toContain(userMediaStream); + expect(mediaHandler.screensharingStreams).toContain(screenSharingStream); + mediaHandler.stopAllStreams(); + expect(mediaHandler.userMediaStreams).not.toContain(userMediaStream); + expect(mediaHandler.screensharingStreams).not.toContain(screenSharingStream); + }); + + it("emits LocalStreamsChanged", async () => { + const onLocalStreamChanged = jest.fn(); + mediaHandler.on(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + mediaHandler.stopAllStreams(); + expect(onLocalStreamChanged).toHaveBeenCalled(); + + mediaHandler.off(MediaHandlerEvent.LocalStreamsChanged, onLocalStreamChanged); + }); + }); +}); diff --git a/src/@types/event.ts b/src/@types/event.ts index 3bb720138d5..168097925b2 100644 --- a/src/@types/event.ts +++ b/src/@types/event.ts @@ -87,6 +87,10 @@ export enum EventType { RoomKeyRequest = "m.room_key_request", ForwardedRoomKey = "m.forwarded_room_key", Dummy = "m.dummy", + + // Group call events + GroupCallPrefix = "org.matrix.msc3401.call", + GroupCallMemberPrefix = "org.matrix.msc3401.call.member", } export enum RelationType { diff --git a/src/client.ts b/src/client.ts index c1b1bb6fe77..316e554c072 100644 --- a/src/client.ts +++ b/src/client.ts @@ -35,6 +35,7 @@ import { StubStore } from "./store/stub"; import { CallEvent, CallEventHandlerMap, createNewMatrixCall, MatrixCall, supportsMatrixCall } from "./webrtc/call"; import { Filter, IFilterDefinition, IRoomEventFilter } from "./filter"; import { CallEventHandlerEvent, CallEventHandler, CallEventHandlerEventHandlerMap } from './webrtc/callEventHandler'; +import { GroupCallEventHandlerEvent, GroupCallEventHandlerEventHandlerMap } from './webrtc/groupCallEventHandler'; import * as utils from './utils'; import { replaceParam, QueryDict, sleep } from './utils'; import { Direction, EventTimeline } from "./models/event-timeline"; @@ -64,12 +65,14 @@ import { FileType, UploadResponse, HTTPError, + IRequestOpts, } from "./http-api"; import { Crypto, CryptoEvent, CryptoEventHandlerMap, fixBackupKey, + ICryptoCallbacks, IBootstrapCrossSigningOpts, ICheckOwnCrossSigningTrustOpts, IMegolmSessionData, @@ -99,29 +102,9 @@ import { } from "./crypto/keybackup"; import { IIdentityServerProvider } from "./@types/IIdentityServerProvider"; import { MatrixScheduler } from "./scheduler"; -import { - IAuthData, - ICryptoCallbacks, - IMinimalEvent, - IRoomEvent, - IStateEvent, - NotificationCountType, - BeaconEvent, - BeaconEventHandlerMap, - RoomEvent, - RoomEventHandlerMap, - RoomMemberEvent, - RoomMemberEventHandlerMap, - RoomStateEvent, - RoomStateEventHandlerMap, - INotificationsResponse, - IFilterResponse, - ITagsResponse, - IStatusResponse, - IPushRule, - PushRuleActionName, - IAuthDict, -} from "./matrix"; +import { BeaconEvent, BeaconEventHandlerMap } from "./models/beacon"; +import { IAuthData, IAuthDict } from "./interactive-auth"; +import { IMinimalEvent, IRoomEvent, IStateEvent } from "./sync-accumulator"; import { CrossSigningKey, IAddSecretStorageKeyOpts, @@ -136,7 +119,9 @@ import { VerificationRequest } from "./crypto/verification/request/VerificationR import { VerificationBase as Verification } from "./crypto/verification/Base"; import * as ContentHelpers from "./content-helpers"; import { CrossSigningInfo, DeviceTrustLevel, ICacheCallbacks, UserTrustLevel } from "./crypto/CrossSigning"; -import { Room, RoomNameState } from "./models/room"; +import { Room, NotificationCountType, RoomEvent, RoomEventHandlerMap, RoomNameState } from "./models/room"; +import { RoomMemberEvent, RoomMemberEventHandlerMap } from "./models/room-member"; +import { RoomStateEvent, RoomStateEventHandlerMap } from "./models/room-state"; import { IAddThreePidOnlyBody, IBindThreePidBody, @@ -153,6 +138,10 @@ import { IRoomDirectoryOptions, ISearchOpts, ISendEventResponse, + INotificationsResponse, + IFilterResponse, + ITagsResponse, + IStatusResponse, } from "./@types/requests"; import { EventType, @@ -184,10 +173,26 @@ import { } from "./@types/search"; import { ISynapseAdminDeactivateResponse, ISynapseAdminWhoisResponse } from "./@types/synapse"; import { IHierarchyRoom } from "./@types/spaces"; -import { IPusher, IPusherRequest, IPushRules, PushRuleAction, PushRuleKind, RuleId } from "./@types/PushRules"; +import { + IPusher, + IPusherRequest, + IPushRule, + IPushRules, + PushRuleAction, + PushRuleActionName, + PushRuleKind, + RuleId, +} from "./@types/PushRules"; import { IThreepid } from "./@types/threepids"; import { CryptoStore } from "./crypto/store/base"; +import { + GroupCall, + IGroupCallDataChannelOptions, + GroupCallIntent, + GroupCallType, +} from "./webrtc/groupCall"; import { MediaHandler } from "./webrtc/mediaHandler"; +import { GroupCallEventHandler } from "./webrtc/groupCallEventHandler"; import { LoginTokenPostResponse, ILoginFlowsResponse, IRefreshTokenResponse, SSOAction } from "./@types/auth"; import { TypedEventEmitter } from "./models/typed-event-emitter"; import { ReceiptType } from "./@types/read_receipts"; @@ -359,6 +364,12 @@ export interface ICreateClientOpts { */ fallbackICEServerAllowed?: boolean; + /** + * If true, to-device signalling for group calls will be encrypted + * with Olm. Default: true. + */ + useE2eForGroupCall?: boolean; + cryptoCallbacks?: ICryptoCallbacks; /** @@ -825,6 +836,7 @@ export enum ClientEvent { DeleteRoom = "deleteRoom", SyncUnexpectedError = "sync.unexpectedError", ClientWellKnown = "WellKnown.client", + ReceivedVoipEvent = "received_voip_event", TurnServers = "turnServers", TurnServersError = "turnServers.error", } @@ -884,6 +896,9 @@ export type EmittedEvents = ClientEvent | UserEvents | CallEvent // re-emitted by call.ts using Object.values | CallEventHandlerEvent.Incoming + | GroupCallEventHandlerEvent.Incoming + | GroupCallEventHandlerEvent.Ended + | GroupCallEventHandlerEvent.Participants | HttpApiEvent.SessionLoggedOut | HttpApiEvent.NoConsent | BeaconEvent; @@ -897,6 +912,7 @@ export type ClientEventHandlerMap = { [ClientEvent.DeleteRoom]: (roomId: string) => void; [ClientEvent.SyncUnexpectedError]: (error: Error) => void; [ClientEvent.ClientWellKnown]: (data: IClientWellKnown) => void; + [ClientEvent.ReceivedVoipEvent]: (event: MatrixEvent) => void; [ClientEvent.TurnServers]: (servers: ITurnServer[]) => void; [ClientEvent.TurnServersError]: (error: Error, fatal: boolean) => void; } & RoomEventHandlerMap @@ -906,6 +922,7 @@ export type ClientEventHandlerMap = { & RoomMemberEventHandlerMap & UserEventHandlerMap & CallEventHandlerEventHandlerMap + & GroupCallEventHandlerEventHandlerMap & CallEventHandlerMap & HttpApiEventHandlerMap & BeaconEventHandlerMap; @@ -936,6 +953,7 @@ export class MatrixClient extends TypedEventEmitter>(); + private useE2eForGroupCall = true; private toDeviceMessageQueue: ToDeviceMessageQueue; // A manager for determining which invites should be ignored. @@ -1004,6 +1024,7 @@ export class MatrixClient extends TypedEventEmitter { + if (this.getGroupCallForRoom(roomId)) { + throw new Error(`${roomId} already has an existing group call`); + } + + const room = this.getRoom(roomId); + + if (!room) { + throw new Error(`Cannot find room ${roomId}`); + } + + return new GroupCall( + this, + room, + type, + isPtt, + intent, + undefined, + dataChannelsEnabled, + dataChannelOptions, + ).create(); + } + + /** + * Wait until an initial state for the given room has been processed by the + * client and the client is aware of any ongoing group calls. Awaiting on + * the promise returned by this method before calling getGroupCallForRoom() + * avoids races where getGroupCallForRoom is called before the state for that + * room has been processed. It does not, however, fix other races, eg. two + * clients both creating a group call at the same time. + * @param roomId The room ID to wait for + * @returns A promise that resolves once existing group calls in the room + * have been processed. + */ + public waitUntilRoomReadyForGroupCalls(roomId: string): Promise { + return this.groupCallEventHandler!.waitUntilRoomReadyForGroupCalls(roomId); + } + + /** + * Get an existing group call for the provided room. + * @param roomId + * @returns {GroupCall} The group call or null if it doesn't already exist. + */ + public getGroupCallForRoom(roomId: string): GroupCall | null { + return this.groupCallEventHandler!.groupCalls.get(roomId) || null; + } + /** * Get the current sync state. * @return {?SyncState} the sync state, which may be null. @@ -3889,9 +3993,8 @@ export class MatrixClient extends TypedEventEmitter { + protected encryptAndSendEvent(room: Room | null, event: MatrixEvent): Promise { let cancelled = false; // Add an extra Promise.resolve() to turn synchronous exceptions into promise rejections, // so that we can handle synchronous and asynchronous exceptions with the @@ -4019,7 +4122,7 @@ export class MatrixClient extends TypedEventEmitter { if (this.isInitialSyncComplete()) { - this.callEventHandler?.start(); + this.callEventHandler!.start(); + this.groupCallEventHandler!.start(); this.off(ClientEvent.Sync, this.startCallEventHandler); } }; @@ -7631,6 +7735,7 @@ export class MatrixClient extends TypedEventEmitter { const pathParams = { $roomId: roomId, @@ -7649,7 +7755,7 @@ export class MatrixClient extends TypedEventEmitter { algorithm?: string; sender_key?: string; } - -interface IEncryptedContent { - algorithm: string; - sender_key: string; - ciphertext: Record; -} /* eslint-enable camelcase */ interface SharedWithData { @@ -1660,6 +1654,9 @@ class MegolmDecryption extends DecryptionAlgorithm { return; } } + + // XXX: switch this to use encryptAndSendToDevices() rather than duplicating it? + await olmlib.ensureOlmSessionsForDevices( this.olmDevice, this.baseApis, { [sender]: [device] }, false, ); @@ -1717,6 +1714,8 @@ class MegolmDecryption extends DecryptionAlgorithm { const deviceInfo = this.crypto.getStoredDevice(userId, deviceId)!; const body = keyRequest.requestBody; + // XXX: switch this to use encryptAndSendToDevices()? + this.olmlib.ensureOlmSessionsForDevices( this.olmDevice, this.baseApis, { [userId]: [deviceInfo], @@ -1910,6 +1909,7 @@ class MegolmDecryption extends DecryptionAlgorithm { for (const [senderKey, sessionId] of sharedHistorySessions) { const payload = await this.buildKeyForwardingMessage(this.roomId, senderKey, sessionId); + // FIXME: use encryptAndSendToDevices() rather than duplicating it here. const promises: Promise[] = []; const contentMap: Record> = {}; for (const [userId, devices] of Object.entries(devicesByUser)) { diff --git a/src/crypto/index.ts b/src/crypto/index.ts index e0e8c01290f..66fdad6683b 100644 --- a/src/crypto/index.ts +++ b/src/crypto/index.ts @@ -127,6 +127,29 @@ export interface IBootstrapCrossSigningOpts { authUploadDeviceSigningKeys?(makeRequest: (authData: any) => Promise<{}>): Promise; } +export interface ICryptoCallbacks { + getCrossSigningKey?: (keyType: string, pubKey: string) => Promise; + saveCrossSigningKeys?: (keys: Record) => void; + shouldUpgradeDeviceVerifications?: ( + users: Record + ) => Promise; + getSecretStorageKey?: ( + keys: {keys: Record}, name: string + ) => Promise<[string, Uint8Array] | null>; + cacheSecretStorageKey?: ( + keyId: string, keyInfo: ISecretStorageKeyInfo, key: Uint8Array + ) => void; + onSecretRequested?: ( + userId: string, deviceId: string, + requestId: string, secretName: string, deviceTrust: DeviceTrustLevel + ) => Promise; + getDehydrationKey?: ( + keyInfo: ISecretStorageKeyInfo, + checkFunc: (key: Uint8Array) => void, + ) => Promise; + getBackupKey?: () => Promise; +} + /* eslint-disable camelcase */ interface IRoomKey { room_id: string; diff --git a/src/embedded.ts b/src/embedded.ts new file mode 100644 index 00000000000..d1dfb50bf2f --- /dev/null +++ b/src/embedded.ts @@ -0,0 +1,357 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + WidgetApi, + WidgetApiToWidgetAction, + MatrixCapabilities, + IWidgetApiRequest, + IWidgetApiAcknowledgeResponseData, + ISendEventToWidgetActionRequest, + ISendToDeviceToWidgetActionRequest, + ISendEventFromWidgetResponseData, +} from "matrix-widget-api"; + +import { IEvent, IContent, EventStatus } from "./models/event"; +import { ISendEventResponse } from "./@types/requests"; +import { EventType } from "./@types/event"; +import { logger } from "./logger"; +import { MatrixClient, ClientEvent, IMatrixClientCreateOpts, IStartClientOpts } from "./client"; +import { SyncApi, SyncState } from "./sync"; +import { SlidingSyncSdk } from "./sliding-sync-sdk"; +import { MatrixEvent } from "./models/event"; +import { User } from "./models/user"; +import { Room } from "./models/room"; +import { ToDeviceBatch } from "./models/ToDeviceMessage"; +import { DeviceInfo } from "./crypto/deviceinfo"; +import { IOlmDevice } from "./crypto/algorithms/megolm"; + +interface IStateEventRequest { + eventType: string; + stateKey?: string; +} + +export interface ICapabilities { + /** + * Event types that this client expects to send. + */ + sendEvent?: string[]; + /** + * Event types that this client expects to receive. + */ + receiveEvent?: string[]; + + /** + * Message types that this client expects to send, or true for all message + * types. + */ + sendMessage?: string[] | true; + /** + * Message types that this client expects to receive, or true for all + * message types. + */ + receiveMessage?: string[] | true; + + /** + * Types of state events that this client expects to send. + */ + sendState?: IStateEventRequest[]; + /** + * Types of state events that this client expects to receive. + */ + receiveState?: IStateEventRequest[]; + + /** + * To-device event types that this client expects to send. + */ + sendToDevice?: string[]; + /** + * To-device event types that this client expects to receive. + */ + receiveToDevice?: string[]; + + /** + * Whether this client needs access to TURN servers. + * @default false + */ + turnServers?: boolean; +} + +/** + * A MatrixClient that routes its requests through the widget API instead of the + * real CS API. + * @experimental This class is considered unstable! + */ +export class RoomWidgetClient extends MatrixClient { + private room?: Room; + private widgetApiReady = new Promise(resolve => this.widgetApi.once("ready", resolve)); + private lifecycle?: AbortController; + private syncState: SyncState | null = null; + + constructor( + private readonly widgetApi: WidgetApi, + private readonly capabilities: ICapabilities, + private readonly roomId: string, + opts: IMatrixClientCreateOpts, + ) { + super(opts); + + // Request capabilities for the functionality this client needs to support + if ( + capabilities.sendEvent?.length + || capabilities.receiveEvent?.length + || capabilities.sendMessage === true + || (Array.isArray(capabilities.sendMessage) && capabilities.sendMessage.length) + || capabilities.receiveMessage === true + || (Array.isArray(capabilities.receiveMessage) && capabilities.receiveMessage.length) + || capabilities.sendState?.length + || capabilities.receiveState?.length + ) { + widgetApi.requestCapabilityForRoomTimeline(roomId); + } + capabilities.sendEvent?.forEach(eventType => + widgetApi.requestCapabilityToSendEvent(eventType), + ); + capabilities.receiveEvent?.forEach(eventType => + widgetApi.requestCapabilityToReceiveEvent(eventType), + ); + if (capabilities.sendMessage === true) { + widgetApi.requestCapabilityToSendMessage(); + } else if (Array.isArray(capabilities.sendMessage)) { + capabilities.sendMessage.forEach(msgType => + widgetApi.requestCapabilityToSendMessage(msgType), + ); + } + if (capabilities.receiveMessage === true) { + widgetApi.requestCapabilityToReceiveMessage(); + } else if (Array.isArray(capabilities.receiveMessage)) { + capabilities.receiveMessage.forEach(msgType => + widgetApi.requestCapabilityToReceiveMessage(msgType), + ); + } + capabilities.sendState?.forEach(({ eventType, stateKey }) => + widgetApi.requestCapabilityToSendState(eventType, stateKey), + ); + capabilities.receiveState?.forEach(({ eventType, stateKey }) => + widgetApi.requestCapabilityToReceiveState(eventType, stateKey), + ); + capabilities.sendToDevice?.forEach(eventType => + widgetApi.requestCapabilityToSendToDevice(eventType), + ); + capabilities.receiveToDevice?.forEach(eventType => + widgetApi.requestCapabilityToReceiveToDevice(eventType), + ); + if (capabilities.turnServers) { + widgetApi.requestCapability(MatrixCapabilities.MSC3846TurnServers); + } + + widgetApi.on(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); + widgetApi.on(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + + // Open communication with the host + widgetApi.start(); + } + + public async startClient(opts: IStartClientOpts = {}): Promise { + this.lifecycle = new AbortController(); + + // Create our own user object artificially (instead of waiting for sync) + // so it's always available, even if the user is not in any rooms etc. + const userId = this.getUserId(); + if (userId) { + this.store.storeUser(new User(userId)); + } + + // Even though we have no access token and cannot sync, the sync class + // still has some valuable helper methods that we make use of, so we + // instantiate it anyways + if (opts.slidingSync) { + this.syncApi = new SlidingSyncSdk(opts.slidingSync, this, opts); + } else { + this.syncApi = new SyncApi(this, opts); + } + + this.room = this.syncApi.createRoom(this.roomId); + this.store.storeRoom(this.room); + + await this.widgetApiReady; + + // Backfill the requested events + // We only get the most recent event for every type + state key combo, + // so it doesn't really matter what order we inject them in + await Promise.all( + this.capabilities.receiveState?.map(async ({ eventType, stateKey }) => { + const rawEvents = await this.widgetApi.readStateEvents(eventType, undefined, stateKey, [this.roomId]); + const events = rawEvents.map(rawEvent => new MatrixEvent(rawEvent as Partial)); + + await this.syncApi!.injectRoomEvents(this.room!, [], events); + events.forEach(event => { + this.emit(ClientEvent.Event, event); + logger.info(`Backfilled event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + }); + }) ?? [], + ); + this.setSyncState(SyncState.Syncing); + logger.info("Finished backfilling events"); + + // Watch for TURN servers, if requested + if (this.capabilities.turnServers) this.watchTurnServers(); + } + + public stopClient() { + this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendEvent}`, this.onEvent); + this.widgetApi.off(`action:${WidgetApiToWidgetAction.SendToDevice}`, this.onToDevice); + + super.stopClient(); + this.lifecycle!.abort(); // Signal to other async tasks that the client has stopped + } + + public async joinRoom(roomIdOrAlias: string): Promise { + if (roomIdOrAlias === this.roomId) return this.room!; + throw new Error(`Unknown room: ${roomIdOrAlias}`); + } + + protected async encryptAndSendEvent(room: Room, event: MatrixEvent): Promise { + let response: ISendEventFromWidgetResponseData; + try { + response = await this.widgetApi.sendRoomEvent(event.getType(), event.getContent(), room.roomId); + } catch (e) { + this.updatePendingEventStatus(room, event, EventStatus.NOT_SENT); + throw e; + } + + room.updatePendingEvent(event, EventStatus.SENT, response.event_id); + return { event_id: response.event_id }; + } + + public async sendStateEvent( + roomId: string, + eventType: string, + content: any, + stateKey = "", + ): Promise { + return await this.widgetApi.sendStateEvent(eventType, stateKey, content, roomId); + } + + public async sendToDevice( + eventType: string, + contentMap: { [userId: string]: { [deviceId: string]: Record } }, + ): Promise<{}> { + await this.widgetApi.sendToDevice(eventType, false, contentMap); + return {}; + } + + public async queueToDevice({ eventType, batch }: ToDeviceBatch): Promise { + const contentMap: { [userId: string]: { [deviceId: string]: object } } = {}; + for (const { userId, deviceId, payload } of batch) { + if (!contentMap[userId]) contentMap[userId] = {}; + contentMap[userId][deviceId] = payload; + } + + await this.widgetApi.sendToDevice(eventType, false, contentMap); + } + + public async encryptAndSendToDevices( + userDeviceInfoArr: IOlmDevice[], + payload: object, + ): Promise { + const contentMap: { [userId: string]: { [deviceId: string]: object } } = {}; + for (const { userId, deviceInfo: { deviceId } } of userDeviceInfoArr) { + if (!contentMap[userId]) contentMap[userId] = {}; + contentMap[userId][deviceId] = payload; + } + + await this.widgetApi.sendToDevice((payload as { type: string }).type, true, contentMap); + } + + // Overridden since we get TURN servers automatically over the widget API, + // and this method would otherwise complain about missing an access token + public async checkTurnServers(): Promise { + return this.turnServers.length > 0; + } + + // Overridden since we 'sync' manually without the sync API + public getSyncState(): SyncState | null { + return this.syncState; + } + + private setSyncState(state: SyncState) { + const oldState = this.syncState; + this.syncState = state; + this.emit(ClientEvent.Sync, state, oldState); + } + + private async ack(ev: CustomEvent): Promise { + await this.widgetApi.transport.reply(ev.detail, {}); + } + + private onEvent = async (ev: CustomEvent) => { + ev.preventDefault(); + + // Verify the room ID matches, since it's possible for the client to + // send us events from other rooms if this widget is always on screen + if (ev.detail.data.room_id === this.roomId) { + const event = new MatrixEvent(ev.detail.data as Partial); + await this.syncApi!.injectRoomEvents(this.room!, [], [event]); + this.emit(ClientEvent.Event, event); + this.setSyncState(SyncState.Syncing); + logger.info(`Received event ${event.getId()} ${event.getType()} ${event.getStateKey()}`); + } else { + const { event_id: eventId, room_id: roomId } = ev.detail.data; + logger.info(`Received event ${eventId} for a different room ${roomId}; discarding`); + } + + await this.ack(ev); + }; + + private onToDevice = async (ev: CustomEvent) => { + ev.preventDefault(); + + const event = new MatrixEvent({ + type: ev.detail.data.type, + sender: ev.detail.data.sender, + content: ev.detail.data.content as IContent, + }); + // Mark the event as encrypted if it was, using fake contents and keys since those are unknown to us + if (ev.detail.data.encrypted) event.makeEncrypted(EventType.RoomMessageEncrypted, {}, "", ""); + + this.emit(ClientEvent.ToDeviceEvent, event); + this.setSyncState(SyncState.Syncing); + await this.ack(ev); + }; + + private async watchTurnServers() { + const servers = this.widgetApi.getTurnServers(); + const onClientStopped = () => servers.return(undefined); + this.lifecycle!.signal.addEventListener("abort", onClientStopped); + + try { + for await (const server of servers) { + this.turnServers = [{ + urls: server.uris, + username: server.username, + credential: server.password, + }]; + this.emit(ClientEvent.TurnServers, this.turnServers); + logger.log(`Received TURN server: ${server.uris}`); + } + } catch (e) { + logger.warn("Error watching TURN servers", e); + } finally { + this.lifecycle!.signal.removeEventListener("abort", onClientStopped); + } + } +} diff --git a/src/http-api/fetch.ts b/src/http-api/fetch.ts index 35698bb62ba..92413c094af 100644 --- a/src/http-api/fetch.ts +++ b/src/http-api/fetch.ts @@ -236,7 +236,7 @@ export class FetchHttpApi { method: Method, url: URL | string, body?: Body, - opts: Pick = {}, + opts: Pick = {}, ): Promise> { const headers = Object.assign({}, opts.headers || {}); const json = opts.json ?? true; @@ -254,6 +254,7 @@ export class FetchHttpApi { } const timeout = opts.localTimeoutMs ?? this.opts.localTimeoutMs; + const keepAlive = opts.keepAlive ?? false; const signals = [ this.abortController.signal, ]; @@ -286,6 +287,7 @@ export class FetchHttpApi { referrerPolicy: "no-referrer", cache: "no-cache", credentials: "omit", // we send credentials via headers + keepalive: keepAlive, }); } catch (e) { if ((e).name === "AbortError") { diff --git a/src/http-api/interface.ts b/src/http-api/interface.ts index c798bec0d6c..37217947027 100644 --- a/src/http-api/interface.ts +++ b/src/http-api/interface.ts @@ -38,6 +38,7 @@ export interface IRequestOpts { headers?: Record; abortSignal?: AbortSignal; localTimeoutMs?: number; + keepAlive?: boolean; // defaults to false json?: boolean; // defaults to true // Set to true to prevent the request function from emitting a Session.logged_out event. diff --git a/src/matrix.ts b/src/matrix.ts index d1999ed85c3..e89feddeb04 100644 --- a/src/matrix.ts +++ b/src/matrix.ts @@ -1,5 +1,5 @@ /* -Copyright 2015-2021 The Matrix.org Foundation C.I.C. +Copyright 2015-2022 The Matrix.org Foundation C.I.C. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,14 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { WidgetApi } from "matrix-widget-api"; + import { MemoryCryptoStore } from "./crypto/store/memory-crypto-store"; import { MemoryStore } from "./store/memory"; import { MatrixScheduler } from "./scheduler"; import { MatrixClient, ICreateClientOpts } from "./client"; -import { DeviceTrustLevel } from "./crypto/CrossSigning"; -import { ISecretStorageKeyInfo } from "./crypto/api"; +import { RoomWidgetClient, ICapabilities } from "./embedded"; export * from "./client"; +export * from "./embedded"; export * from "./http-api"; export * from "./autodiscovery"; export * from "./sync-accumulator"; @@ -51,9 +53,16 @@ export * from './@types/requests'; export * from './@types/search'; export * from './models/room-summary'; export * as ContentHelpers from "./content-helpers"; +export type { ICryptoCallbacks } from "./crypto"; // used to be located here +export { createNewMatrixCall } from "./webrtc/call"; +export type { MatrixCall } from "./webrtc/call"; export { - createNewMatrixCall, -} from "./webrtc/call"; + GroupCallEvent, + GroupCallIntent, + GroupCallState, + GroupCallType, +} from "./webrtc/groupCall"; +export type { GroupCall } from "./webrtc/groupCall"; let cryptoStoreFactory = () => new MemoryCryptoStore; @@ -67,34 +76,20 @@ export function setCryptoStoreFactory(fac) { cryptoStoreFactory = fac; } -export interface ICryptoCallbacks { - getCrossSigningKey?: (keyType: string, pubKey: string) => Promise; - saveCrossSigningKeys?: (keys: Record) => void; - shouldUpgradeDeviceVerifications?: ( - users: Record - ) => Promise; - getSecretStorageKey?: ( - keys: {keys: Record}, name: string - ) => Promise<[string, Uint8Array] | null>; - cacheSecretStorageKey?: ( - keyId: string, keyInfo: ISecretStorageKeyInfo, key: Uint8Array - ) => void; - onSecretRequested?: ( - userId: string, deviceId: string, - requestId: string, secretName: string, deviceTrust: DeviceTrustLevel - ) => Promise; - getDehydrationKey?: ( - keyInfo: ISecretStorageKeyInfo, - checkFunc: (key: Uint8Array) => void, - ) => Promise; - getBackupKey?: () => Promise; +function amendClientOpts(opts: ICreateClientOpts): ICreateClientOpts { + opts.store = opts.store ?? new MemoryStore({ + localStorage: global.localStorage, + }); + opts.scheduler = opts.scheduler ?? new MatrixScheduler(); + opts.cryptoStore = opts.cryptoStore ?? cryptoStoreFactory(); + + return opts; } /** * Construct a Matrix Client. Similar to {@link module:client.MatrixClient} * except that the 'request', 'store' and 'scheduler' dependencies are satisfied. - * @param {(Object)} opts The configuration options for this client. If - * this is a string, it is assumed to be the base URL. These configuration + * @param {Object} opts The configuration options for this client. These configuration * options will be passed directly to {@link module:client.MatrixClient}. * @param {Object} opts.store If not set, defaults to * {@link module:store/memory.MemoryStore}. @@ -111,33 +106,15 @@ export interface ICryptoCallbacks { * @see {@link module:client.MatrixClient} for the full list of options for * opts. */ -export function createClient(opts: ICreateClientOpts) { - opts.store = opts.store || new MemoryStore({ - localStorage: global.localStorage, - }); - opts.scheduler = opts.scheduler || new MatrixScheduler(); - opts.cryptoStore = opts.cryptoStore || cryptoStoreFactory(); - return new MatrixClient(opts); +export function createClient(opts: ICreateClientOpts): MatrixClient { + return new MatrixClient(amendClientOpts(opts)); } -/** - * A wrapper for the request function interface. - * @callback requestWrapperFunction - * @param {requestFunction} origRequest The underlying request function being - * wrapped - * @param {Object} opts The options for this HTTP request, given in the same - * form as {@link requestFunction}. - * @param {requestCallback} callback The request callback. - */ - -/** - * The request callback interface for performing HTTP requests. This matches the - * API for the {@link https://github.com/request/request#requestoptions-callback| - * request NPM module}. The SDK will implement a callback which meets this - * interface in order to handle the HTTP response. - * @callback requestCallback - * @param {Error} err The error if one occurred, else falsey. - * @param {Object} response The HTTP response which consists of - * {statusCode: {Number}, headers: {Object}} - * @param {Object} body The parsed HTTP response body. - */ +export function createRoomWidgetClient( + widgetApi: WidgetApi, + capabilities: ICapabilities, + roomId: string, + opts: ICreateClientOpts, +): MatrixClient { + return new RoomWidgetClient(widgetApi, capabilities, roomId, amendClientOpts(opts)); +} diff --git a/src/models/beacon.ts b/src/models/beacon.ts index 79876c469f9..fc817b05e24 100644 --- a/src/models/beacon.ts +++ b/src/models/beacon.ts @@ -16,7 +16,7 @@ limitations under the License. import { MBeaconEventContent } from "../@types/beacon"; import { BeaconInfoState, BeaconLocationState, parseBeaconContent, parseBeaconInfoContent } from "../content-helpers"; -import { MatrixEvent } from "../matrix"; +import { MatrixEvent } from "./event"; import { sortEventsByLatestContentTimestamp } from "../utils"; import { TypedEventEmitter } from "./typed-event-emitter"; diff --git a/src/models/read-receipt.ts b/src/models/read-receipt.ts index 4e864aa5634..b28976c5c6b 100644 --- a/src/models/read-receipt.ts +++ b/src/models/read-receipt.ts @@ -12,9 +12,11 @@ limitations under the License. */ import { ReceiptType } from "../@types/read_receipts"; -import { EventTimelineSet, EventType, MatrixEvent } from "../matrix"; import { ListenerMap, TypedEventEmitter } from "./typed-event-emitter"; import * as utils from "../utils"; +import { MatrixEvent } from "./event"; +import { EventType } from "../@types/event"; +import { EventTimelineSet } from "./event-timeline-set"; export const MAIN_ROOM_TIMELINE = "main"; diff --git a/src/models/thread.ts b/src/models/thread.ts index 0e3b6c3b51c..384500aed4f 100644 --- a/src/models/thread.ts +++ b/src/models/thread.ts @@ -16,12 +16,13 @@ limitations under the License. import { Optional } from "matrix-events-sdk"; -import { MatrixClient, MatrixEventEvent, RelationType, RoomEvent } from "../matrix"; +import { MatrixClient } from "../client"; import { TypedReEmitter } from "../ReEmitter"; -import { IThreadBundledRelationship, MatrixEvent } from "./event"; +import { RelationType } from "../@types/event"; +import { IThreadBundledRelationship, MatrixEvent, MatrixEventEvent } from "./event"; import { EventTimeline } from "./event-timeline"; import { EventTimelineSet, EventTimelineSetHandlerMap } from './event-timeline-set'; -import { Room } from './room'; +import { Room, RoomEvent } from './room'; import { RoomState } from "./room-state"; import { ServerControlledNamespacedValue } from "../NamespacedValue"; import { logger } from "../logger"; diff --git a/src/sliding-sync-sdk.ts b/src/sliding-sync-sdk.ts index ed7ef9cb5f7..962d824a8e5 100644 --- a/src/sliding-sync-sdk.ts +++ b/src/sliding-sync-sdk.ts @@ -33,8 +33,11 @@ import { SlidingSyncEvent, SlidingSyncState, } from "./sliding-sync"; -import { EventType, IPushRules } from "./matrix"; +import { EventType } from "./@types/event"; +import { IPushRules } from "./@types/PushRules"; import { PushProcessor } from "./pushprocessor"; +import { RoomStateEvent } from "./models/room-state"; +import { RoomMemberEvent } from "./models/room-member"; // Number of consecutive failed syncs that will lead to a syncState of ERROR as opposed // to RECONNECTING. This is needed to inform the client of server issues when the @@ -389,6 +392,60 @@ export class SlidingSyncSdk { return this.syncStateData ?? null; } + // Helper functions which set up JS SDK structs are below and are identical to the sync v2 counterparts + + public createRoom(roomId: string): Room { // XXX cargoculted from sync.ts + const { timelineSupport } = this.client; + const room = new Room(roomId, this.client, this.client.getUserId()!, { + lazyLoadMembers: this.opts.lazyLoadMembers, + pendingEventOrdering: this.opts.pendingEventOrdering, + timelineSupport, + }); + this.client.reEmitter.reEmit(room, [ + RoomEvent.Name, + RoomEvent.Redaction, + RoomEvent.RedactionCancelled, + RoomEvent.Receipt, + RoomEvent.Tags, + RoomEvent.LocalEchoUpdated, + RoomEvent.AccountData, + RoomEvent.MyMembership, + RoomEvent.Timeline, + RoomEvent.TimelineReset, + ]); + this.registerStateListeners(room); + return room; + } + + private registerStateListeners(room: Room): void { // XXX cargoculted from sync.ts + // we need to also re-emit room state and room member events, so hook it up + // to the client now. We need to add a listener for RoomState.members in + // order to hook them correctly. + this.client.reEmitter.reEmit(room.currentState, [ + RoomStateEvent.Events, + RoomStateEvent.Members, + RoomStateEvent.NewMember, + RoomStateEvent.Update, + ]); + room.currentState.on(RoomStateEvent.NewMember, (event, state, member) => { + member.user = this.client.getUser(member.userId) ?? undefined; + this.client.reEmitter.reEmit(member, [ + RoomMemberEvent.Name, + RoomMemberEvent.Typing, + RoomMemberEvent.PowerLevel, + RoomMemberEvent.Membership, + ]); + }); + } + + /* + private deregisterStateListeners(room: Room): void { // XXX cargoculted from sync.ts + // could do with a better way of achieving this. + room.currentState.removeAllListeners(RoomStateEvent.Events); + room.currentState.removeAllListeners(RoomStateEvent.Members); + room.currentState.removeAllListeners(RoomStateEvent.NewMember); + } */ + private shouldAbortSync(error: MatrixError): boolean { if (error.errcode === "M_UNKNOWN_TOKEN") { // The logout already happened, we just need to stop. @@ -484,7 +541,7 @@ export class SlidingSyncSdk { if (roomData.invite_state) { const inviteStateEvents = mapEvents(this.client, room.roomId, roomData.invite_state); - this.processRoomEvents(room, inviteStateEvents); + this.injectRoomEvents(room, inviteStateEvents); if (roomData.initial) { room.recalculate(); this.client.store.storeRoom(room); @@ -552,10 +609,11 @@ export class SlidingSyncSdk { // reason to stop incrementally tracking notifications and // reset the timeline. this.client.resetNotifTimelineSet(); + this.registerStateListeners(room); } } */ - this.processRoomEvents(room, stateEvents, timelineEvents, false); + this.injectRoomEvents(room, stateEvents, timelineEvents, false); // we deliberately don't add ephemeral events to the timeline room.addEphemeralEvents(ephemeralEvents); @@ -594,6 +652,7 @@ export class SlidingSyncSdk { } /** + * Injects events into a room's model. * @param {Room} room * @param {MatrixEvent[]} stateEventList A list of state events. This is the state * at the *START* of the timeline list if it is supplied. @@ -601,7 +660,7 @@ export class SlidingSyncSdk { * @param {boolean} fromCache whether the sync response came from cache * is earlier in time. Higher index is later. */ - private processRoomEvents( + public injectRoomEvents( room: Room, stateEventList: MatrixEvent[], timelineEventList?: MatrixEvent[], @@ -822,7 +881,6 @@ function ensureNameEvent(client: MatrixClient, roomId: string, roomData: MSC3575 // Helper functions which set up JS SDK structs are below and are identical to the sync v2 counterparts, // just outside the class. - function mapEvents(client: MatrixClient, roomId: string | undefined, events: object[], decrypt = true): MatrixEvent[] { const mapper = client.getEventMapper({ decrypt }); return (events as Array).map(function(e) { diff --git a/src/sync.ts b/src/sync.ts index 183e74a613e..59798c9e09e 100644 --- a/src/sync.ts +++ b/src/sync.ts @@ -334,7 +334,7 @@ export class SyncApi { // events so that clients can start back-paginating. room.getLiveTimeline().setPaginationToken(leaveObj.timeline.prev_batch, EventTimeline.BACKWARDS); - await this.processRoomEvents(room, stateEvents, events); + await this.injectRoomEvents(room, stateEvents, events); room.recalculate(); client.store.storeRoom(room); @@ -367,7 +367,7 @@ export class SyncApi { response.messages.chunk = response.messages.chunk || []; response.state = response.state || []; - // FIXME: Mostly duplicated from processRoomEvents but not entirely + // FIXME: Mostly duplicated from injectRoomEvents but not entirely // because "state" in this API is at the BEGINNING of the chunk const oldStateEvents = utils.deepCopy(response.state) .map(client.getEventMapper()); @@ -1207,7 +1207,7 @@ export class SyncApi { const room = inviteObj.room; const stateEvents = this.mapSyncEventsFormat(inviteObj.invite_state, room); - await this.processRoomEvents(room, stateEvents); + await this.injectRoomEvents(room, stateEvents); const inviter = room.currentState.getStateEvents(EventType.RoomMember, client.getUserId()!)?.getSender(); @@ -1363,7 +1363,7 @@ export class SyncApi { } try { - await this.processRoomEvents(room, stateEvents, events, syncEventData.fromCache); + await this.injectRoomEvents(room, stateEvents, events, syncEventData.fromCache); } catch (e) { logger.error(`Failed to process events on room ${room.roomId}:`, e); } @@ -1418,7 +1418,7 @@ export class SyncApi { const events = this.mapSyncEventsFormat(leaveObj.timeline, room); const accountDataEvents = this.mapSyncEventsFormat(leaveObj.account_data); - await this.processRoomEvents(room, stateEvents, events); + await this.injectRoomEvents(room, stateEvents, events); room.addAccountData(accountDataEvents); room.recalculate(); @@ -1656,14 +1656,15 @@ export class SyncApi { } /** + * Injects events into a room's model. * @param {Room} room * @param {MatrixEvent[]} stateEventList A list of state events. This is the state * at the *START* of the timeline list if it is supplied. * @param {MatrixEvent[]} [timelineEventList] A list of timeline events, including threaded. Lower index - * @param {boolean} fromCache whether the sync response came from cache * is earlier in time. Higher index is later. + * @param {boolean} fromCache whether the sync response came from cache */ - private async processRoomEvents( + public async injectRoomEvents( room: Room, stateEventList: MatrixEvent[], timelineEventList?: MatrixEvent[], diff --git a/src/webrtc/audioContext.ts b/src/webrtc/audioContext.ts new file mode 100644 index 00000000000..8a9ceb15da6 --- /dev/null +++ b/src/webrtc/audioContext.ts @@ -0,0 +1,44 @@ +/* +Copyright 2022 The Matrix.org Foundation C.I.C. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +let audioContext: AudioContext | null = null; +let refCount = 0; + +/** + * Acquires a reference to the shared AudioContext. + * It's highly recommended to reuse this AudioContext rather than creating your + * own, because multiple AudioContexts can be problematic in some browsers. + * Make sure to call releaseContext when you're done using it. + * @returns {AudioContext} The shared AudioContext + */ +export const acquireContext = (): AudioContext => { + if (audioContext === null) audioContext = new AudioContext(); + refCount++; + return audioContext; +}; + +/** + * Signals that one of the references to the shared AudioContext has been + * released, allowing the context and associated hardware resources to be + * cleaned up if nothing else is using it. + */ +export const releaseContext = () => { + refCount--; + if (refCount === 0) { + audioContext?.close(); + audioContext = null; + } +}; diff --git a/src/webrtc/call.ts b/src/webrtc/call.ts index 2a11de5756b..69077643a01 100644 --- a/src/webrtc/call.ts +++ b/src/webrtc/call.ts @@ -22,6 +22,8 @@ limitations under the License. * @module webrtc/call */ +import { parse as parseSdp, write as writeSdp } from "sdp-transform"; + import { logger } from '../logger'; import * as utils from '../utils'; import { MatrixEvent } from '../models/event'; @@ -45,8 +47,10 @@ import { } from './callEventTypes'; import { CallFeed } from './callFeed'; import { MatrixClient } from "../client"; -import { ISendEventResponse } from "../@types/requests"; import { EventEmitterEvents, TypedEventEmitter } from "../models/typed-event-emitter"; +import { DeviceInfo } from '../crypto/deviceinfo'; +import { GroupCallUnknownDeviceError } from './groupCall'; +import { IScreensharingOpts } from "./mediaHandler"; import { MatrixError } from "../http-api"; // events: hangup, error(err), replaced(call), state(state, oldState) @@ -68,10 +72,14 @@ import { MatrixError } from "../http-api"; */ interface CallOpts { - roomId: string; + roomId?: string; + invitee?: string; client: MatrixClient; forceTURN?: boolean; turnServers?: Array; + opponentDeviceId?: string; + opponentSessionId?: string; + groupCallId?: string; } interface TurnServer { @@ -86,6 +94,24 @@ interface AssertedIdentity { displayName: string; } +enum MediaType { + AUDIO = "audio", + VIDEO = "video", +} + +enum CodecName { + OPUS = "opus", + // add more as needed +} + +// Used internally to specify modifications to codec parameters in SDP +interface CodecParamsMod { + mediaType: MediaType; + codec: CodecName; + enableDtx?: boolean; // true to enable discontinuous transmission, false to disable, undefined to leave as-is + maxAverageBitrate?: number; // sets the max average bitrate, or undefined to leave as-is +} + export enum CallState { Fledgling = 'fledgling', InviteSent = 'invite_sent', @@ -133,6 +159,8 @@ export enum CallEvent { LengthChanged = 'length_changed', DataChannel = 'datachannel', + + SendVoipEvent = "send_voip_event", } export enum CallErrorCode { @@ -164,6 +192,11 @@ export enum CallErrorCode { */ CreateAnswer = 'create_answer', + /** + * An offer could not be created + */ + CreateOffer = 'create_offer', + /** * Error code used when we fail to send the answer * for some reason other than there being unknown devices @@ -214,6 +247,11 @@ export enum CallErrorCode { * We transferred the call off to somewhere else */ Transfered = 'transferred', + + /** + * A call from the same user was found with a new session id + */ + NewSession = 'new_session', } /** @@ -238,10 +276,23 @@ export class CallError extends Error { } } -function genCallID(): string { +export function genCallID(): string { return Date.now().toString() + randomString(16); } +function getCodecParamMods(isPtt: boolean): CodecParamsMod[] { + const mods = [ + { + mediaType: "audio", + codec: "opus", + enableDtx: true, + maxAverageBitrate: isPtt ? 12000 : undefined, + }, + ] as CodecParamsMod[]; + + return mods; +} + export type CallEventHandlerMap = { [CallEvent.DataChannel]: (channel: RTCDataChannel) => void; [CallEvent.FeedsChanged]: (feeds: CallFeed[]) => void; @@ -251,12 +302,23 @@ export type CallEventHandlerMap = { [CallEvent.LocalHoldUnhold]: (onHold: boolean) => void; [CallEvent.LengthChanged]: (length: number) => void; [CallEvent.State]: (state: CallState, oldState?: CallState) => void; - [CallEvent.Hangup]: () => void; + [CallEvent.Hangup]: (call: MatrixCall) => void; [CallEvent.AssertedIdentityChanged]: () => void; /* @deprecated */ [CallEvent.HoldUnhold]: (onHold: boolean) => void; + [CallEvent.SendVoipEvent]: (event: Record) => void; }; +// The key of the transceiver map (purpose + media type, separated by ':') +type TransceiverKey = string; + +// generates keys for the map of transceivers +// kind is unfortunately a string rather than MediaType as this is the type of +// track.kind +function getTransceiverKey(purpose: SDPStreamMetadataPurpose, kind: TransceiverKey): string { + return purpose + ':' + kind; +} + /** * Construct a new Matrix Call. * @constructor @@ -269,13 +331,20 @@ export type CallEventHandlerMap = { * @param {MatrixClient} opts.client The Matrix Client instance to send events to. */ export class MatrixCall extends TypedEventEmitter { - public roomId: string; + public roomId?: string; public callId: string; + public invitee?: string; public state = CallState.Fledgling; public hangupParty?: CallParty; public hangupReason?: string; public direction?: CallDirection; public ourPartyId: string; + public peerConn?: RTCPeerConnection; + public toDeviceSeq = 0; + + // whether this call should have push-to-talk semantics + // This should be set by the consumer on incoming & outgoing calls. + public isPtt = false; private readonly client: MatrixClient; private readonly forceTURN?: boolean; @@ -285,21 +354,24 @@ export class MatrixCall extends TypedEventEmitter = []; private candidateSendTries = 0; - private sentEndOfCandidates = false; - private peerConn?: RTCPeerConnection; + private candidatesEnded = false; private feeds: Array = []; - private usermediaSenders: Array = []; - private screensharingSenders: Array = []; + + // our transceivers for each purpose and type of media + private transceivers = new Map(); + private inviteOrAnswerSent = false; - private waitForLocalAVStream?: boolean; + private waitForLocalAVStream = false; private successor?: MatrixCall; private opponentMember?: RoomMember; private opponentVersion?: number | string; // The party ID of the other side: undefined if we haven't chosen a partner // yet, null if we have but they didn't send a party ID. - private opponentPartyId?: string | null; + private opponentPartyId: string | null | undefined; private opponentCaps?: CallCapabilities; + private iceDisconnectedTimeout?: ReturnType; private inviteTimeout?: ReturnType; + private readonly removeTrackListeners = new Map void>(); // The logic of when & if a call is on hold is nontrivial and explained in is*OnHold // This flag represents whether we want the other party to be on hold @@ -312,7 +384,9 @@ export class MatrixCall extends TypedEventEmitter; // If candidates arrive before we've picked an opponent (which, in particular, // will happen if the opponent sends candidates eagerly before the user answers @@ -325,12 +399,25 @@ export class MatrixCall extends TypedEventEmitter; private callLength = 0; + private opponentDeviceId?: string; + private opponentDeviceInfo?: DeviceInfo; + private opponentSessionId?: string; + public groupCallId?: string; + constructor(opts: CallOpts) { super(); + this.roomId = opts.roomId; + this.invitee = opts.invitee; this.client = opts.client; - this.forceTURN = opts.forceTURN; - this.ourPartyId = this.client.deviceId!; + + if (!this.client.deviceId) throw new Error("Client must have a device ID to start calls"); + + this.forceTURN = opts.forceTURN ?? false; + this.ourPartyId = this.client.deviceId; + this.opponentDeviceId = opts.opponentDeviceId; + this.opponentSessionId = opts.opponentSessionId; + this.groupCallId = opts.groupCallId; // Array of Objects with urls, username, credential keys this.turnServers = opts.turnServers || []; if (this.turnServers.length === 0 && this.client.isFallbackICEServerAllowed()) { @@ -365,10 +452,9 @@ export class MatrixCall extends TypedEventEmitter !feed.isLocal()); } + private async initOpponentCrypto() { + if (!this.opponentDeviceId) return; + if (!this.client.getUseE2eForGroupCall()) return; + // It's possible to want E2EE and yet not have the means to manage E2EE + // ourselves (for example if the client is a RoomWidgetClient) + if (!this.client.isCryptoEnabled()) { + // All we know is the device ID + this.opponentDeviceInfo = new DeviceInfo(this.opponentDeviceId); + return; + } + // if we've got to this point, we do want to init crypto, so throw if we can't + if (!this.client.crypto) throw new Error("Crypto is not initialised."); + + const userId = this.invitee || this.getOpponentMember()?.userId; + + if (!userId) throw new Error("Couldn't find opponent user ID to init crypto"); + + const deviceInfoMap = await this.client.crypto.deviceList.downloadKeys([userId], false); + this.opponentDeviceInfo = deviceInfoMap[userId][this.opponentDeviceId]; + if (this.opponentDeviceInfo === undefined) { + throw new GroupCallUnknownDeviceError(userId); + } + } + /** * Generates and returns localSDPStreamMetadata * @returns {SDPStreamMetadata} localSDPStreamMetadata */ - private getLocalSDPStreamMetadata(): SDPStreamMetadata { + private getLocalSDPStreamMetadata(updateStreamIds = false): SDPStreamMetadata { const metadata: SDPStreamMetadata = {}; for (const localFeed of this.getLocalFeeds()) { - metadata[localFeed.stream.id] = { + if (updateStreamIds) { + localFeed.sdpMetadataStreamId = localFeed.stream.id; + } + + metadata[localFeed.sdpMetadataStreamId] = { purpose: localFeed.purpose, audio_muted: localFeed.isAudioMuted(), video_muted: localFeed.isVideoMuted(), }; } - logger.debug("Got local SDPStreamMetadata", metadata); return metadata; } @@ -513,7 +630,8 @@ export class MatrixCall extends TypedEventEmitter callFeed.stream.id === feed.stream.id)) { + logger.info(`Ignoring duplicate local stream ${callFeed.stream.id} in call ${this.callId}`); + return; + } + this.feeds.push(callFeed); if (addToPeerConnection) { - const senderArray = callFeed.purpose === SDPStreamMetadataPurpose.Usermedia ? - this.usermediaSenders : this.screensharingSenders; - // Empty the array - senderArray.splice(0, senderArray.length); - for (const track of callFeed.stream.getTracks()) { logger.info( + `Call ${this.callId} ` + `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + @@ -625,14 +750,47 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender); + if (newTransciever) { + this.transceivers.set(tKey, newTransciever); + } else { + logger.warn("Didn't find a matching transceiver after adding track!"); + } + } } } logger.info( - `Pushed local stream ` + - `(id="${callFeed.stream.id}", ` + - `active="${callFeed.stream.active}", ` + + `Call ${this.callId} ` + + `Pushed local stream `+ + `(id="${callFeed.stream.id}", `+ + `active="${callFeed.stream.active}", `+ `purpose="${callFeed.purpose}")`, ); @@ -645,21 +803,31 @@ export class MatrixCall extends TypedEventEmitter { + // Time out the call if it's ringing for too long + const ringingTimer = setTimeout(() => { if (this.state == CallState.Ringing) { - logger.debug("Call invite has expired. Hanging up."); + logger.debug(`Call ${this.callId} invite has expired. Hanging up.`); this.hangupParty = CallParty.Remote; // effectively this.setState(CallState.Ended); this.stopAllMedia(); - if (this.peerConn?.signalingState != 'closed') { - this.peerConn?.close(); + if (this.peerConn!.signalingState != 'closed') { + this.peerConn!.close(); } - this.emit(CallEvent.Hangup); + this.emit(CallEvent.Hangup, this); } }, invite.lifetime - event.getLocalAge()); + + const onState = (state: CallState) => { + if (state !== CallState.Ringing) { + clearTimeout(ringingTimer); + this.off(CallEvent.State, onState); + } + }; + this.on(CallEvent.State, onState); } } @@ -781,12 +961,13 @@ export class MatrixCall extends TypedEventEmitter feed.clone())); + } } this.successor = newCall; this.emit(CallEvent.Replaced, newCall); @@ -915,7 +1100,8 @@ export class MatrixCall extends TypedEventEmittererror), ); @@ -976,36 +1163,46 @@ export class MatrixCall extends TypedEventEmitter { + public async setScreensharingEnabled(enabled: boolean, opts?: IScreensharingOpts): Promise { // Skip if there is nothing to do if (enabled && this.isScreensharing()) { - logger.warn(`There is already a screensharing stream - there is nothing to do!`); + logger.warn(`Call ${this.callId} There is already a screensharing stream - there is nothing to do!`); return true; } else if (!enabled && !this.isScreensharing()) { - logger.warn(`There already isn't a screensharing stream - there is nothing to do!`); + logger.warn(`Call ${this.callId} There already isn't a screensharing stream - there is nothing to do!`); return false; } // Fallback to replaceTrack() if (!this.opponentSupportsSDPStreamMetadata()) { - return this.setScreensharingEnabledWithoutMetadataSupport(enabled, desktopCapturerSourceId); + return this.setScreensharingEnabledWithoutMetadataSupport(enabled, opts); } - logger.debug(`Set screensharing enabled? ${enabled}`); + logger.debug(`Call ${this.callId} set screensharing enabled? ${enabled}`); if (enabled) { try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + const stream = await this.client.getMediaHandler().getScreensharingStream(opts); if (!stream) return false; this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare); return true; } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); + logger.error(`Call ${this.callId} Failed to get screen-sharing stream:`, err); return false; } } else { - for (const sender of this.screensharingSenders) { - this.peerConn?.removeTrack(sender); + const audioTransceiver = this.transceivers.get(getTransceiverKey( + SDPStreamMetadataPurpose.Screenshare, "audio", + )); + const videoTransceiver = this.transceivers.get(getTransceiverKey( + SDPStreamMetadataPurpose.Screenshare, "video", + )); + + for (const transceiver of [audioTransceiver, videoTransceiver]) { + // this is slightly mixing the track and transceiver API but is basically just shorthand + // for removing the sender. + if (transceiver && transceiver.sender) this.peerConn!.removeTrack(transceiver.sender); } + this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!); this.deleteFeedByStream(this.localScreensharingStream!); return false; @@ -1020,28 +1217,34 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug(`Set screensharing enabled? ${enabled} using replaceTrack()`); + logger.debug(`Call ${this.callId} Set screensharing enabled? ${enabled} using replaceTrack()`); if (enabled) { try { - const stream = await this.client.getMediaHandler().getScreensharingStream(desktopCapturerSourceId); + const stream = await this.client.getMediaHandler().getScreensharingStream(opts); if (!stream) return false; const track = stream.getTracks().find(track => track.kind === "video"); - const sender = this.usermediaSenders.find(sender => sender.track?.kind === "video"); + + const sender = this.transceivers.get(getTransceiverKey( + SDPStreamMetadataPurpose.Usermedia, "video", + ))?.sender; + sender?.replaceTrack(track ?? null); this.pushNewLocalFeed(stream, SDPStreamMetadataPurpose.Screenshare, false); return true; } catch (err) { - logger.error("Failed to get screen-sharing stream:", err); + logger.error(`Call ${this.callId} Failed to get screen-sharing stream:`, err); return false; } } else { const track = this.localUsermediaStream?.getTracks().find((track) => track.kind === "video"); - const sender = this.usermediaSenders.find((sender) => sender.track?.kind === "video"); + const sender = this.transceivers.get(getTransceiverKey( + SDPStreamMetadataPurpose.Usermedia, "video", + ))?.sender; sender?.replaceTrack(track ?? null); this.client.getMediaHandler().stopScreensharingStream(this.localScreensharingStream!); @@ -1061,6 +1264,8 @@ export class MatrixCall extends TypedEventEmitter sender.track?.kind === track.kind); - let newSender: RTCRtpSender; + const tKey = getTransceiverKey(SDPStreamMetadataPurpose.Usermedia, track.kind); + const oldSender = this.transceivers.get(tKey)?.sender; + let added = false; if (oldSender) { + try { + logger.info( + `Call ${this.callId} `+ + `Replacing track (` + + `id="${track.id}", ` + + `kind="${track.kind}", ` + + `streamId="${stream.id}", ` + + `streamPurpose="${callFeed.purpose}"` + + `) to peer connection`, + ); + await oldSender.replaceTrack(track); + added = true; + } catch (error) { + logger.warn(`replaceTrack failed: adding new transceiver instead`, error); + } + } + + if (!added) { logger.info( - `Replacing track (` + - `id="${track.id}", ` + - `kind="${track.kind}", ` + - `streamId="${stream.id}", ` + - `streamPurpose="${callFeed.purpose}"` + - `) to peer connection`, - ); - await oldSender.replaceTrack(track); - newSender = oldSender; - } else { - logger.info( + `Call ${this.callId} `+ `Adding track (` + `id="${track.id}", ` + `kind="${track.kind}", ` + @@ -1099,13 +1311,16 @@ export class MatrixCall extends TypedEventEmitter t.sender === newSender); + if (newTransciever) { + this.transceivers.set(tKey, newTransciever); + } else { + logger.warn("Couldn't find matching transceiver for newly added track!"); + } + } } - - this.usermediaSenders = newSenders; } /** @@ -1114,6 +1329,7 @@ export class MatrixCall extends TypedEventEmitter { + logger.log(`call ${this.callId} setLocalVideoMuted ${muted}`); if (!await this.client.getMediaHandler().hasVideoDevice()) { return this.isLocalVideoMuted(); } @@ -1124,6 +1340,7 @@ export class MatrixCall extends TypedEventEmitter { + logger.log(`call ${this.callId} setMicrophoneMuted ${muted}`); if (!await this.client.getMediaHandler().hasAudioDevice()) { return this.isMicrophoneMuted(); } @@ -1156,6 +1374,7 @@ export class MatrixCall extends TypedEventEmitter { + await this.sendVoipEvent(EventType.CallSDPStreamMetadataChangedPrefix, { + [SDPStreamMetadataKey]: this.getLocalSDPStreamMetadata(), + }); + } + + private gotCallFeedsForInvite(callFeeds: CallFeed[], requestScreenshareFeed = false): void { if (this.successor) { - this.successor.gotCallFeedsForAnswer(callFeeds); + this.successor.queueGotCallFeedsForAnswer(callFeeds); return; } if (this.callHasEnded()) { @@ -1257,9 +1481,15 @@ export class MatrixCall extends TypedEventEmitter this.gotCallFeedsForAnswer(callFeeds)); + } else { + this.responsePromiseChain = this.gotCallFeedsForAnswer(callFeeds); + } + } + + // Enables DTX (discontinuous transmission) on the given session to reduce + // bandwidth when transmitting silence + private mungeSdp(description: RTCSessionDescriptionInit, mods: CodecParamsMod[]): void { + // The only way to enable DTX at this time is through SDP munging + const sdp = parseSdp(description.sdp!); + + sdp.media.forEach(media => { + const payloadTypeToCodecMap = new Map(); + const codecToPayloadTypeMap = new Map(); + for (const rtp of media.rtp) { + payloadTypeToCodecMap.set(rtp.payload, rtp.codec); + codecToPayloadTypeMap.set(rtp.codec, rtp.payload); + } + + for (const mod of mods) { + if (mod.mediaType !== media.type) continue; + + if (!codecToPayloadTypeMap.has(mod.codec)) { + logger.info(`Ignoring SDP modifications for ${mod.codec} as it's not present.`); + continue; + } + + const extraconfig: string[] = []; + if (mod.enableDtx !== undefined) { + extraconfig.push(`usedtx=${mod.enableDtx ? '1' : '0'}`); + } + if (mod.maxAverageBitrate !== undefined) { + extraconfig.push(`maxaveragebitrate=${mod.maxAverageBitrate}`); + } + + let found = false; + for (const fmtp of media.fmtp) { + if (payloadTypeToCodecMap.get(fmtp.payload) === mod.codec) { + found = true; + fmtp.config += ";" + extraconfig.join(";"); + } + } + if (!found) { + media.fmtp.push({ + payload: codecToPayloadTypeMap.get(mod.codec)!, + config: extraconfig.join(";"), + }); + } + } + }); + description.sdp = writeSdp(sdp); + } + + private async createOffer(): Promise { + const offer = await this.peerConn!.createOffer(); + this.mungeSdp(offer, getCodecParamMods(this.isPtt)); + return offer; + } + + private async createAnswer(): Promise { + const answer = await this.peerConn!.createAnswer(); + this.mungeSdp(answer, getCodecParamMods(this.isPtt)); + return answer; + } + private async gotCallFeedsForAnswer(callFeeds: CallFeed[]): Promise { if (this.callHasEnded()) return; @@ -1321,18 +1621,22 @@ export class MatrixCall extends TypedEventEmitter { if (event.candidate) { + if (this.candidatesEnded) { + logger.warn("Got candidate after candidates have ended - ignoring!"); + return; + } + logger.debug( "Call " + this.callId + " got local ICE " + event.candidate.sdpMid + " candidate: " + event.candidate.candidate, @@ -1363,29 +1675,18 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug("ice gathering state changed to " + this.peerConn?.iceGatheringState); - if (this.peerConn?.iceGatheringState === 'complete' && !this.sentEndOfCandidates) { - // If we didn't get an empty-string candidate to signal the end of candidates, - // create one ourselves now gathering has finished. - // We cast because the interface lists all the properties as required but we - // only want to send 'candidate' - // XXX: We probably want to send either sdpMid or sdpMLineIndex, as it's not strictly - // correct to have a candidate that lacks both of these. We'd have to figure out what - // previous candidates had been sent with and copy them. - const c = { - candidate: '', - } as RTCIceCandidate; - this.queueCandidate(c); - this.sentEndOfCandidates = true; + logger.debug(`Call ${this.callId} ice gathering state changed to ${this.peerConn!.iceGatheringState}`); + if (this.peerConn?.iceGatheringState === 'complete') { + this.queueCandidate(null); } }; @@ -1398,7 +1699,7 @@ export class MatrixCall extends TypedEventEmitter(); const candidates = content.candidates; if (!candidates) { - logger.info("Ignoring candidates event with no candidates!"); + logger.info(`Call ${this.callId} Ignoring candidates event with no candidates!`); return; } @@ -1406,15 +1707,18 @@ export class MatrixCall extends TypedEventEmitter { if (this.direction !== CallDirection.Inbound) { - logger.warn("Got select_answer for an outbound call: ignoring"); + logger.warn(`Call ${this.callId} Got select_answer for an outbound call: ignoring`); return; } const selectedPartyId = event.getContent().selected_party_id; if (selectedPartyId === undefined || selectedPartyId === null) { - logger.warn("Got nonsensical select_answer with null/undefined selected_party_id: ignoring"); + logger.warn(`Call ${ + this.callId} Got nonsensical select_answer with null/undefined selected_party_id: ignoring`); return; } if (selectedPartyId !== this.ourPartyId) { - logger.info(`Got select_answer for party ID ${selectedPartyId}: we are party ID ${this.ourPartyId}.`); + logger.info(`Call ${this.callId} Got select_answer for party ID ${ + selectedPartyId}: we are party ID ${this.ourPartyId}.`); // The other party has picked somebody else's answer await this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); } @@ -1506,7 +1813,7 @@ export class MatrixCall extends TypedEventEmitter(); const description = content.description; if (!description || !description.sdp || !description.type) { - logger.info("Ignoring invalid m.call.negotiate event"); + logger.info(`Call ${this.callId} Ignoring invalid m.call.negotiate event`); return; } // Politeness always follows the direction of the call: in a glare situation, @@ -1523,7 +1830,7 @@ export class MatrixCall extends TypedEventEmitter => { - logger.debug("Created offer: ", description); + private queueGotLocalOffer(): void { + // Ensure only one negotiate/answer event is being processed at a time. + if (this.responsePromiseChain) { + this.responsePromiseChain = + this.responsePromiseChain.then(() => this.wrappedGotLocalOffer()); + } else { + this.responsePromiseChain = this.wrappedGotLocalOffer(); + } + } + + private async wrappedGotLocalOffer(): Promise { + this.makingOffer = true; + try { + await this.gotLocalOffer(); + } catch (e) { + this.getLocalOfferFailed(e as Error); + return; + } finally { + this.makingOffer = false; + } + } + + private async gotLocalOffer(): Promise { + logger.debug(`Call ${this.callId} Setting local description`); if (this.callHasEnded()) { logger.debug("Ignoring newly created offer on call ID " + this.callId + @@ -1605,10 +1942,20 @@ export class MatrixCall extends TypedEventEmitter { - logger.error("Failed to get local offer", err); + logger.error(`Call ${this.callId} Failed to get local offer`, err); this.emit( CallEvent.Error, @@ -1704,7 +2055,7 @@ export class MatrixCall extends TypedEventEmitter void) | null) { + this.candidatesEnded = false; + this.peerConn!.restartIce(); + } else { + this.hangup(CallErrorCode.IceFailed, false); + } + } else if (this.peerConn?.iceConnectionState == 'disconnected') { + this.iceDisconnectedTimeout = setTimeout(() => { + this.hangup(CallErrorCode.IceFailed, false); + }, 30 * 1000); + } + + // In PTT mode, override feed status to muted when we lose connection to + // the peer, since we don't want to block the line if they're not saying anything. + // Experimenting in Chrome, this happens after 5 or 6 seconds, which is probably + // fast enough. + if (this.isPtt && ["failed", "disconnected"].includes(this.peerConn!.iceConnectionState)) { + for (const feed of this.getRemoteFeeds()) { + feed.setAudioVideoMuted(true, true); + } } }; @@ -1746,18 +2121,25 @@ export class MatrixCall extends TypedEventEmitter { if (ev.streams.length === 0) { - logger.warn(`Streamless ${ev.track.kind} found: ignoring.`); + logger.warn(`Call ${this.callId} Streamless ${ev.track.kind} found: ignoring.`); return; } const stream = ev.streams[0]; this.pushRemoteFeed(stream); - stream.addEventListener("removetrack", () => { - if (stream.getTracks().length === 0) { - logger.info(`Stream ID ${stream.id} has no tracks remaining - removing`); - this.deleteFeedByStream(stream); - } - }); + + if (!this.removeTrackListeners.has(stream)) { + const onRemoveTrack = () => { + if (stream.getTracks().length === 0) { + logger.info(`Call ${this.callId} removing track streamId: ${stream.id}`); + this.deleteFeedByStream(stream); + stream.removeEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.delete(stream); + } + }; + stream.addEventListener("removetrack", onRemoveTrack); + this.removeTrackListeners.set(stream, onRemoveTrack); + } }; private onDataChannel = (ev: RTCDataChannelEvent): void => { @@ -1792,38 +2174,22 @@ export class MatrixCall extends TypedEventEmitter => { - logger.info("Negotiation is needed!"); + logger.info(`Call ${this.callId} Negotiation is needed!`); if (this.state !== CallState.CreateOffer && this.opponentVersion === 0) { - logger.info("Opponent does not support renegotiation: ignoring negotiationneeded event"); + logger.info(`Call ${ + this.callId} Opponent does not support renegotiation: ignoring negotiationneeded event`); return; } - this.makingOffer = true; - try { - this.getRidOfRTXCodecs(); - const myOffer = await this.peerConn!.createOffer(); - await this.gotLocalOffer(myOffer); - } catch (e) { - this.getLocalOfferFailed(e); - return; - } finally { - this.makingOffer = false; - } + this.queueGotLocalOffer(); }; public onHangupReceived = (msg: MCallHangupReject): void => { @@ -1835,7 +2201,8 @@ export class MatrixCall extends TypedEventEmitter { - logger.debug("Call ID " + this.callId + " answered elsewhere"); + logger.debug("Call " + this.callId + " answered elsewhere"); this.terminate(CallParty.Remote, CallErrorCode.AnsweredElsewhere, true); }; @@ -1878,15 +2245,67 @@ export class MatrixCall extends TypedEventEmitter { - return this.client.sendEvent(this.roomId, eventType, Object.assign({}, content, { + private async sendVoipEvent(eventType: string, content: object): Promise { + const realContent = Object.assign({}, content, { version: VOIP_PROTO_VERSION, call_id: this.callId, party_id: this.ourPartyId, - })); + conf_id: this.groupCallId, + }); + + if (this.opponentDeviceId) { + const toDeviceSeq = this.toDeviceSeq++; + const content = { + ...realContent, + device_id: this.client.deviceId, + sender_session_id: this.client.getSessionId(), + dest_session_id: this.opponentSessionId, + seq: toDeviceSeq, + }; + + this.emit(CallEvent.SendVoipEvent, { + type: "toDevice", + eventType, + userId: this.invitee || this.getOpponentMember()?.userId, + opponentDeviceId: this.opponentDeviceId, + content, + }); + + const userId = this.invitee || this.getOpponentMember()!.userId; + if (this.client.getUseE2eForGroupCall()) { + await this.client.encryptAndSendToDevices([{ + userId, + deviceInfo: this.opponentDeviceInfo!, + }], { + type: eventType, + content, + }); + } else { + await this.client.sendToDevice(eventType, { + [userId]: { + [this.opponentDeviceId]: content, + }, + }); + } + } else { + this.emit(CallEvent.SendVoipEvent, { + type: "sendEvent", + eventType, + roomId: this.roomId, + content: realContent, + userId: this.invitee || this.getOpponentMember()?.userId, + }); + + await this.client.sendEvent(this.roomId!, eventType, realContent); + } } - private queueCandidate(content: RTCIceCandidate): void { + /** + * Queue a candidate to be sent + * @param content The candidate to queue up, or null if candidates have finished being generated + * and end-of-candidates should be signalled + */ + private queueCandidate(content: RTCIceCandidate | null): void { // We partially de-trickle candidates by waiting for `delay` before sending them // amalgamated, in order to avoid sending too many m.call.candidates events and hitting // rate limits in Matrix. @@ -1896,7 +2315,11 @@ export class MatrixCall extends TypedEventEmitter { if (this.callHasEnded()) return; - this.callStatsAtEnd = await this.collectCallStats(); + this.hangupParty = hangupParty; + this.hangupReason = hangupReason; + this.setState(CallState.Ended); if (this.inviteTimeout) { clearTimeout(this.inviteTimeout); @@ -1998,31 +2444,47 @@ export class MatrixCall extends TypedEventEmitter { - if (this.candidateSendQueue.length === 0) { + if (this.candidateSendQueue.length === 0 || this.callHasEnded()) { return; } const candidates = this.candidateSendQueue; this.candidateSendQueue = []; ++this.candidateSendTries; - const content = { - candidates: candidates, - }; - logger.debug("Attempting to send " + candidates.length + " candidates"); + const content = { candidates: candidates.map(candidate => candidate.toJSON()) }; + if (this.candidatesEnded) { + // If there are no more candidates, signal this by adding an empty string candidate + content.candidates.push({ + candidate: '', + }); + } + logger.debug(`Call ${this.callId} attempting to send ${candidates.length} candidates`); try { await this.sendVoipEvent(EventType.CallCandidates, content); // reset our retry count if we have successfully sent our candidates // otherwise queueCandidate() will refuse to try to flush the queue this.candidateSendTries = 0; + + // Try to send candidates again just in case we received more candidates while sending. + this.sendCandidateQueue(); } catch (error) { // don't retry this event: we'll send another one later as we might // have more candidates by then. @@ -2066,8 +2535,9 @@ export class MatrixCall extends TypedEventEmitter 5) { logger.debug( - "Failed to send candidates on attempt " + this.candidateSendTries + - ". Giving up on this call.", error, + `Call ${this.callId} failed to send candidates on attempt ${ + this.candidateSendTries}. Giving up on this call.`, + error, ); const code = CallErrorCode.SignallingFailed; @@ -2081,7 +2551,7 @@ export class MatrixCall extends TypedEventEmitter { this.sendCandidateQueue(); }, delayMs); @@ -2129,10 +2599,12 @@ export class MatrixCall extends TypedEventEmitter { + public async placeCallWithCallFeeds(callFeeds: CallFeed[], requestScreenshareFeed = false): Promise { this.checkForErrorListener(); this.direction = CallDirection.Outbound; + await this.initOpponentCrypto(); + // XXX Find a better way to do this this.client.callEventHandler!.calls.set(this.callId, this); @@ -2140,13 +2612,13 @@ export class MatrixCall extends TypedEventEmitter(); - logger.debug(`Choosing party ID ${msg.party_id} for call ID ${this.callId}`); + logger.debug(`Call ${this.callId} choosing opponent party ID ${msg.party_id}`); this.opponentVersion = msg.version; if (this.opponentVersion === 0) { @@ -2197,13 +2670,14 @@ export class MatrixCall extends TypedEventEmitter { const bufferedCandidates = this.remoteCandidateBuffer.get(this.opponentPartyId!); if (bufferedCandidates) { - logger.info(`Adding ${bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); + logger.info(`Call ${this.callId} Adding ${ + bufferedCandidates.length} buffered candidates for opponent ${this.opponentPartyId}`); await this.addIceCandidates(bufferedCandidates); } this.remoteCandidateBuffer.clear(); @@ -2215,17 +2689,17 @@ export class MatrixCall extends TypedEventEmitter, enabled: boolean): void { +export function setTracksEnabled(tracks: Array, enabled: boolean): void { for (const track of tracks) { track.enabled = enabled; } @@ -2289,18 +2763,22 @@ export function supportsMatrixCall(): boolean { export function createNewMatrixCall( client: MatrixClient, roomId: string, - options?: Pick, + options?: Pick, ): MatrixCall | null { if (!supportsMatrixCall()) return null; const optionsForceTURN = options ? options.forceTURN : false; - const opts = { + const opts: CallOpts = { client: client, roomId: roomId, + invitee: options?.invitee, turnServers: client.getTurnServers(), // call level options forceTURN: client.forceTURN || optionsForceTURN, + opponentDeviceId: options?.opponentDeviceId, + opponentSessionId: options?.opponentSessionId, + groupCallId: options?.groupCallId, }; const call = new MatrixCall(opts); diff --git a/src/webrtc/callEventHandler.ts b/src/webrtc/callEventHandler.ts index cb132e8fef6..c4c48cf5058 100644 --- a/src/webrtc/callEventHandler.ts +++ b/src/webrtc/callEventHandler.ts @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { MatrixEvent, MatrixEventEvent } from '../models/event'; +import { MatrixEvent } from '../models/event'; import { logger } from '../logger'; -import { CallDirection, CallErrorCode, CallState, createNewMatrixCall, MatrixCall } from './call'; +import { CallDirection, CallError, CallErrorCode, CallState, createNewMatrixCall, MatrixCall } from './call'; import { EventType } from '../@types/event'; import { ClientEvent, MatrixClient } from '../client'; import { MCallAnswer, MCallHangupReject } from "./callEventTypes"; -import { SyncState } from "../sync"; +import { GroupCall, GroupCallErrorCode, GroupCallEvent, GroupCallUnknownDeviceError } from './groupCall'; import { RoomEvent } from "../models/room"; // Don't ring unless we'd be ringing for at least 3 seconds: the user needs some @@ -36,10 +36,15 @@ export type CallEventHandlerEventHandlerMap = { }; export class CallEventHandler { - client: MatrixClient; - calls: Map; - callEventBuffer: MatrixEvent[]; - candidateEventsByCall: Map>; + // XXX: Most of these are only public because of the tests + public calls: Map; + public callEventBuffer: MatrixEvent[]; + public nextSeqByCall: Map = new Map(); + public toDeviceEventBuffers: Map> = new Map(); + + private client: MatrixClient; + private candidateEventsByCall: Map>; + private eventBufferPromiseChain?: Promise; constructor(client: MatrixClient) { this.client = client; @@ -57,90 +62,166 @@ export class CallEventHandler { } public start() { - this.client.on(ClientEvent.Sync, this.evaluateEventBuffer); + this.client.on(ClientEvent.Sync, this.onSync); this.client.on(RoomEvent.Timeline, this.onRoomTimeline); + this.client.on(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); } public stop() { - this.client.removeListener(ClientEvent.Sync, this.evaluateEventBuffer); + this.client.removeListener(ClientEvent.Sync, this.onSync); this.client.removeListener(RoomEvent.Timeline, this.onRoomTimeline); + this.client.removeListener(ClientEvent.ToDeviceEvent, this.onToDeviceEvent); } - private evaluateEventBuffer = async () => { - if (this.client.getSyncState() === SyncState.Syncing) { - await Promise.all(this.callEventBuffer.map(event => { - this.client.decryptEventIfNeeded(event); - })); - - const ignoreCallIds = new Set(); - // inspect the buffer and mark all calls which have been answered - // or hung up before passing them to the call event handler. - for (const ev of this.callEventBuffer) { - if (ev.getType() === EventType.CallAnswer || ev.getType() === EventType.CallHangup) { - ignoreCallIds.add(ev.getContent().call_id); - } + private onSync = (): void => { + // Process the current event buffer and start queuing into a new one. + const currentEventBuffer = this.callEventBuffer; + this.callEventBuffer = []; + + // Ensure correct ordering by only processing this queue after the previous one has finished processing + if (this.eventBufferPromiseChain) { + this.eventBufferPromiseChain = + this.eventBufferPromiseChain.then(() => this.evaluateEventBuffer(currentEventBuffer)); + } else { + this.eventBufferPromiseChain = this.evaluateEventBuffer(currentEventBuffer); + } + }; + + private async evaluateEventBuffer(eventBuffer: MatrixEvent[]) { + await Promise.all(eventBuffer.map((event) => this.client.decryptEventIfNeeded(event))); + + const callEvents = eventBuffer.filter((event) => { + const eventType = event.getType(); + return eventType.startsWith("m.call.") || eventType.startsWith("org.matrix.call."); + }); + + const ignoreCallIds = new Set(); + + // inspect the buffer and mark all calls which have been answered + // or hung up before passing them to the call event handler. + for (const event of callEvents) { + const eventType = event.getType(); + + if (eventType=== EventType.CallAnswer || eventType === EventType.CallHangup) { + ignoreCallIds.add(event.getContent().call_id); } - // now loop through the buffer chronologically and inject them - for (const e of this.callEventBuffer) { - if (e.getType() === EventType.CallInvite && ignoreCallIds.has(e.getContent().call_id)) { - // This call has previously been answered or hung up: ignore it - continue; - } - try { - await this.handleCallEvent(e); - } catch (e) { - logger.error("Caught exception handling call event", e); - } + } + + // Process call events in the order that they were received + for (const event of callEvents) { + const eventType = event.getType(); + const callId = event.getContent().call_id; + + if (eventType === EventType.CallInvite && ignoreCallIds.has(callId)) { + // This call has previously been answered or hung up: ignore it + continue; + } + + try { + await this.handleCallEvent(event); + } catch (e) { + logger.error("Caught exception handling call event", e); } - this.callEventBuffer = []; } - }; + } private onRoomTimeline = (event: MatrixEvent) => { - this.client.decryptEventIfNeeded(event); - // any call events or ones that might be once they're decrypted - if (this.eventIsACall(event) || event.isBeingDecrypted()) { - // queue up for processing once all events from this sync have been - // processed (see above). + this.callEventBuffer.push(event); + }; + + private onToDeviceEvent = (event: MatrixEvent): void => { + const content = event.getContent(); + + if (!content.call_id) { this.callEventBuffer.push(event); + return; } - if (event.isBeingDecrypted() || event.isDecryptionFailure()) { - // add an event listener for once the event is decrypted. - event.once(MatrixEventEvent.Decrypted, async () => { - if (!this.eventIsACall(event)) return; + if (!this.nextSeqByCall.has(content.call_id)) { + this.nextSeqByCall.set(content.call_id, 0); + } - if (this.callEventBuffer.includes(event)) { - // we were waiting for that event to decrypt, so recheck the buffer - this.evaluateEventBuffer(); - } else { - // This one wasn't buffered so just run the event handler for it - // straight away - try { - await this.handleCallEvent(event); - } catch (e) { - logger.error("Caught exception handling call event", e); - } - } - }); + if (content.seq === undefined) { + this.callEventBuffer.push(event); + return; } - }; - private eventIsACall(event: MatrixEvent): boolean { - const type = event.getType(); - /** - * Unstable prefixes: - * - org.matrix.call. : MSC3086 https://github.com/matrix-org/matrix-doc/pull/3086 - */ - return type.startsWith("m.call.") || type.startsWith("org.matrix.call."); - } + const nextSeq = this.nextSeqByCall.get(content.call_id) || 0; + + if (content.seq !== nextSeq) { + if (!this.toDeviceEventBuffers.has(content.call_id)) { + this.toDeviceEventBuffers.set(content.call_id, []); + } + + const buffer = this.toDeviceEventBuffers.get(content.call_id)!; + const index = buffer.findIndex((e) => e.getContent().seq > content.seq); + + if (index === -1) { + buffer.push(event); + } else { + buffer.splice(index, 0, event); + } + } else { + const callId = content.call_id; + this.callEventBuffer.push(event); + this.nextSeqByCall.set(callId, content.seq + 1); + + const buffer = this.toDeviceEventBuffers.get(callId); + + let nextEvent = buffer && buffer.shift(); + + while (nextEvent && nextEvent.getContent().seq === this.nextSeqByCall.get(callId)) { + this.callEventBuffer.push(nextEvent); + this.nextSeqByCall.set(callId, nextEvent.getContent().seq + 1); + nextEvent = buffer!.shift(); + } + } + }; private async handleCallEvent(event: MatrixEvent) { + this.client.emit(ClientEvent.ReceivedVoipEvent, event); + const content = event.getContent(); + const callRoomId = ( + event.getRoomId() || + this.client.groupCallEventHandler!.getGroupCallById(content.conf_id)?.room?.roomId + ); + const groupCallId = content.conf_id; const type = event.getType() as EventType; - const weSentTheEvent = event.getSender() === this.client.credentials.userId; + const senderId = event.getSender()!; + const weSentTheEvent = senderId === this.client.credentials.userId; let call = content.call_id ? this.calls.get(content.call_id) : undefined; + let opponentDeviceId: string | undefined; + + let groupCall: GroupCall | undefined; + if (groupCallId) { + groupCall = this.client.groupCallEventHandler!.getGroupCallById(groupCallId); + + if (!groupCall) { + logger.warn(`Cannot find a group call ${groupCallId} for event ${type}. Ignoring event.`); + return; + } + + opponentDeviceId = content.device_id; + + if (!opponentDeviceId) { + logger.warn(`Cannot find a device id for ${senderId}. Ignoring event.`); + groupCall.emit( + GroupCallEvent.Error, + new GroupCallUnknownDeviceError(senderId), + ); + return; + } + + if (content.dest_session_id !== this.client.getSessionId()) { + logger.warn("Call event does not match current session id, ignoring."); + return; + } + } + + if (!callRoomId) return; + if (type === EventType.CallInvite) { // ignore invites you send if (weSentTheEvent) return; @@ -156,12 +237,20 @@ export class CallEventHandler { ); } - const timeUntilTurnCresExpire = this.client.getTurnServersExpiry() - Date.now(); + if (content.invitee && content.invitee !== this.client.getUserId()) { + return; // This invite was meant for another user in the room + } + + const timeUntilTurnCresExpire = (this.client.getTurnServersExpiry() ?? 0) - Date.now(); logger.info("Current turn creds expire in " + timeUntilTurnCresExpire + " ms"); call = createNewMatrixCall( this.client, - event.getRoomId()!, - { forceTURN: this.client.forceTURN }, + callRoomId, + { + forceTURN: this.client.forceTURN, opponentDeviceId, + groupCallId, + opponentSessionId: content.sender_session_id, + }, ) ?? undefined; if (!call) { logger.log( @@ -175,7 +264,17 @@ export class CallEventHandler { } call.callId = content.call_id; - await call.initWithInvite(event); + try { + await call.initWithInvite(event); + } catch (e) { + if (e instanceof CallError) { + if (e.code === GroupCallErrorCode.UnknownDevice) { + groupCall?.emit(GroupCallEvent.Error, e); + } else { + logger.error(e); + } + } + } this.calls.set(call.callId, call); // if we stashed candidate events for that call ID, play them back now @@ -195,6 +294,7 @@ export class CallEventHandler { if ( call.roomId === thisCall.roomId && thisCall.direction === CallDirection.Outbound && + call.getOpponentMember()?.userId === thisCall.invitee && isCalling ) { existingCall = thisCall; @@ -203,21 +303,12 @@ export class CallEventHandler { } if (existingCall) { - // If we've only got to wait_local_media or create_offer and - // we've got an invite, pick the incoming call because we know - // we haven't sent our invite yet otherwise, pick whichever - // call has the lowest call ID (by string comparison) - if ( - existingCall.state === CallState.WaitLocalMedia || - existingCall.state === CallState.CreateOffer || - existingCall.callId > call.callId - ) { + if (existingCall.callId > call.callId) { logger.log( "Glare detected: answering incoming call " + call.callId + " and canceling outgoing call " + existingCall.callId, ); existingCall.replacedBy(call); - call.answer(); } else { logger.log( "Glare detected: rejecting incoming call " + call.callId + @@ -249,7 +340,14 @@ export class CallEventHandler { // if not live, store the fact that the call has ended because // we're probably getting events backwards so // the hangup will come before the invite - call = createNewMatrixCall(this.client, event.getRoomId()!) ?? undefined; + call = createNewMatrixCall( + this.client, + callRoomId, + { + opponentDeviceId, + opponentSessionId: content.sender_session_id, + }, + ) ?? undefined; if (call) { call.callId = content.call_id; call.initWithHangup(event); diff --git a/src/webrtc/callEventTypes.ts b/src/webrtc/callEventTypes.ts index c8ad4b8272f..4f43a70a1d5 100644 --- a/src/webrtc/callEventTypes.ts +++ b/src/webrtc/callEventTypes.ts @@ -36,6 +36,8 @@ export interface MCallBase { call_id: string; version: string | number; party_id?: string; + sender_session_id?: string; + dest_session_id?: string; } export interface MCallAnswer extends MCallBase { @@ -53,6 +55,9 @@ export interface MCallInviteNegotiate extends MCallBase { description: RTCSessionDescription; lifetime: number; capabilities?: CallCapabilities; + invitee?: string; + sender_session_id?: string; + dest_session_id?: string; [SDPStreamMetadataKey]: SDPStreamMetadata; } diff --git a/src/webrtc/callFeed.ts b/src/webrtc/callFeed.ts index 5718d4e8d6a..965a0444123 100644 --- a/src/webrtc/callFeed.ts +++ b/src/webrtc/callFeed.ts @@ -15,8 +15,10 @@ limitations under the License. */ import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { acquireContext, releaseContext } from "./audioContext"; import { MatrixClient } from "../client"; import { RoomMember } from "../models/room-member"; +import { logger } from "../logger"; import { TypedEventEmitter } from "../models/typed-event-emitter"; const POLLING_INTERVAL = 200; // ms @@ -25,7 +27,7 @@ const SPEAKING_SAMPLE_COUNT = 8; // samples export interface ICallFeedOpts { client: MatrixClient; - roomId: string; + roomId?: string; userId: string; stream: MediaStream; purpose: SDPStreamMetadataPurpose; @@ -42,27 +44,33 @@ export interface ICallFeedOpts { export enum CallFeedEvent { NewStream = "new_stream", MuteStateChanged = "mute_state_changed", + LocalVolumeChanged = "local_volume_changed", VolumeChanged = "volume_changed", Speaking = "speaking", + Disposed = "disposed", } type EventHandlerMap = { [CallFeedEvent.NewStream]: (stream: MediaStream) => void; [CallFeedEvent.MuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; + [CallFeedEvent.LocalVolumeChanged]: (localVolume: number) => void; [CallFeedEvent.VolumeChanged]: (volume: number) => void; [CallFeedEvent.Speaking]: (speaking: boolean) => void; + [CallFeedEvent.Disposed]: () => void; }; export class CallFeed extends TypedEventEmitter { public stream: MediaStream; + public sdpMetadataStreamId: string; public userId: string; public purpose: SDPStreamMetadataPurpose; public speakingVolumeSamples: number[]; private client: MatrixClient; - private roomId: string; + private roomId?: string; private audioMuted: boolean; private videoMuted: boolean; + private localVolume = 1; private measuringVolumeActivity = false; private audioContext?: AudioContext; private analyser?: AnalyserNode; @@ -70,6 +78,7 @@ export class CallFeed extends TypedEventEmitter private speakingThreshold = SPEAKING_THRESHOLD; private speaking = false; private volumeLooperTimeout?: ReturnType; + private _disposed = false; constructor(opts: ICallFeedOpts) { super(); @@ -81,6 +90,7 @@ export class CallFeed extends TypedEventEmitter this.audioMuted = opts.audioMuted; this.videoMuted = opts.videoMuted; this.speakingVolumeSamples = new Array(SPEAKING_SAMPLE_COUNT).fill(-Infinity); + this.sdpMetadataStreamId = opts.stream.id; this.updateStream(null, opts.stream); this.stream = opts.stream; // updateStream does this, but this makes TS happier @@ -115,10 +125,8 @@ export class CallFeed extends TypedEventEmitter } private initVolumeMeasuring(): void { - const AudioContext = window.AudioContext || window.webkitAudioContext; - if (!this.hasAudioTrack || !AudioContext) return; - - this.audioContext = new AudioContext(); + if (!this.hasAudioTrack) return; + if (!this.audioContext) this.audioContext = acquireContext(); this.analyser = this.audioContext.createAnalyser(); this.analyser.fftSize = 512; @@ -174,6 +182,17 @@ export class CallFeed extends TypedEventEmitter return this.speaking; } + /** + * Replaces the current MediaStream with a new one. + * The stream will be different and new stream as remore parties are + * concerned, but this can be used for convenience locally to set up + * volume listeners automatically on the new stream etc. + * @param newStream new stream with which to replace the current one + */ + public setNewStream(newStream: MediaStream): void { + this.updateStream(this.stream, newStream); + } + /** * Set one or both of feed's internal audio and video video mute state * Either value may be null to leave it as-is @@ -197,7 +216,7 @@ export class CallFeed extends TypedEventEmitter */ public measureVolumeActivity(enabled: boolean): void { if (enabled) { - if (!this.audioContext || !this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; + if (!this.analyser || !this.frequencyBinCount || !this.hasAudioTrack) return; this.measuringVolumeActivity = true; this.volumeLooper(); @@ -248,7 +267,54 @@ export class CallFeed extends TypedEventEmitter this.volumeLooperTimeout = setTimeout(this.volumeLooper, POLLING_INTERVAL); }; + public clone(): CallFeed { + const mediaHandler = this.client.getMediaHandler(); + const stream = this.stream.clone(); + logger.log(`callFeed cloning stream ${this.stream.id} newStream ${stream.id}`); + + if (this.purpose === SDPStreamMetadataPurpose.Usermedia) { + mediaHandler.userMediaStreams.push(stream); + } else { + mediaHandler.screensharingStreams.push(stream); + } + + return new CallFeed({ + client: this.client, + roomId: this.roomId, + userId: this.userId, + stream, + purpose: this.purpose, + audioMuted: this.audioMuted, + videoMuted: this.videoMuted, + }); + } + public dispose(): void { clearTimeout(this.volumeLooperTimeout); + this.stream?.removeEventListener("addtrack", this.onAddTrack); + if (this.audioContext) { + this.audioContext = undefined; + this.analyser = undefined; + releaseContext(); + } + this._disposed = true; + this.emit(CallFeedEvent.Disposed); + } + + public get disposed(): boolean { + return this._disposed; + } + + private set disposed(value: boolean) { + this._disposed = value; + } + + public getLocalVolume(): number { + return this.localVolume; + } + + public setLocalVolume(localVolume: number): void { + this.localVolume = localVolume; + this.emit(CallFeedEvent.LocalVolumeChanged, localVolume); } } diff --git a/src/webrtc/groupCall.ts b/src/webrtc/groupCall.ts new file mode 100644 index 00000000000..e20d388a2bc --- /dev/null +++ b/src/webrtc/groupCall.ts @@ -0,0 +1,1304 @@ +import { TypedEventEmitter } from "../models/typed-event-emitter"; +import { CallFeed, SPEAKING_THRESHOLD } from "./callFeed"; +import { MatrixClient } from "../client"; +import { CallErrorCode, + CallEvent, + CallEventHandlerMap, + CallState, + genCallID, + MatrixCall, + setTracksEnabled, + createNewMatrixCall, + CallError, +} from "./call"; +import { RoomMember } from "../models/room-member"; +import { Room } from "../models/room"; +import { logger } from "../logger"; +import { ReEmitter } from "../ReEmitter"; +import { SDPStreamMetadataPurpose } from "./callEventTypes"; +import { ISendEventResponse } from "../@types/requests"; +import { MatrixEvent } from "../models/event"; +import { EventType } from "../@types/event"; +import { CallEventHandlerEvent } from "./callEventHandler"; +import { GroupCallEventHandlerEvent } from "./groupCallEventHandler"; +import { IScreensharingOpts } from "./mediaHandler"; + +export enum GroupCallIntent { + Ring = "m.ring", + Prompt = "m.prompt", + Room = "m.room", +} + +export enum GroupCallType { + Video = "m.video", + Voice = "m.voice", +} + +export enum GroupCallTerminationReason { + CallEnded = "call_ended", +} + +export enum GroupCallEvent { + GroupCallStateChanged = "group_call_state_changed", + ActiveSpeakerChanged = "active_speaker_changed", + CallsChanged = "calls_changed", + UserMediaFeedsChanged = "user_media_feeds_changed", + ScreenshareFeedsChanged = "screenshare_feeds_changed", + LocalScreenshareStateChanged = "local_screenshare_state_changed", + LocalMuteStateChanged = "local_mute_state_changed", + ParticipantsChanged = "participants_changed", + Error = "error", +} + +export type GroupCallEventHandlerMap = { + [GroupCallEvent.GroupCallStateChanged]: (newState: GroupCallState, oldState: GroupCallState) => void; + [GroupCallEvent.ActiveSpeakerChanged]: (activeSpeaker: string) => void; + [GroupCallEvent.CallsChanged]: (calls: MatrixCall[]) => void; + [GroupCallEvent.UserMediaFeedsChanged]: (feeds: CallFeed[]) => void; + [GroupCallEvent.ScreenshareFeedsChanged]: (feeds: CallFeed[]) => void; + [GroupCallEvent.LocalScreenshareStateChanged]: ( + isScreensharing: boolean, feed?: CallFeed, sourceId?: string, + ) => void; + [GroupCallEvent.LocalMuteStateChanged]: (audioMuted: boolean, videoMuted: boolean) => void; + [GroupCallEvent.ParticipantsChanged]: (participants: RoomMember[]) => void; + [GroupCallEvent.Error]: (error: GroupCallError) => void; +}; + +export enum GroupCallErrorCode { + NoUserMedia = "no_user_media", + UnknownDevice = "unknown_device", + PlaceCallFailed = "place_call_failed" +} + +export class GroupCallError extends Error { + public code: string; + + constructor(code: GroupCallErrorCode, msg: string, err?: Error) { + // Still don't think there's any way to have proper nested errors + if (err) { + super(msg + ": " + err); + } else { + super(msg); + } + + this.code = code; + } +} + +export class GroupCallUnknownDeviceError extends GroupCallError { + constructor(public userId: string) { + super(GroupCallErrorCode.UnknownDevice, "No device found for " + userId); + } +} + +export class OtherUserSpeakingError extends Error { + constructor() { + super("Cannot unmute: another user is speaking"); + } +} + +export interface IGroupCallDataChannelOptions { + ordered: boolean; + maxPacketLifeTime: number; + maxRetransmits: number; + protocol: string; +} + +export interface IGroupCallRoomMemberFeed { + purpose: SDPStreamMetadataPurpose; + // TODO: Sources for adaptive bitrate +} + +export interface IGroupCallRoomMemberDevice { + "device_id": string; + "session_id": string; + "feeds": IGroupCallRoomMemberFeed[]; +} + +export interface IGroupCallRoomMemberCallState { + "m.call_id": string; + "m.foci"?: string[]; + "m.devices": IGroupCallRoomMemberDevice[]; +} + +export interface IGroupCallRoomMemberState { + "m.calls": IGroupCallRoomMemberCallState[]; + "m.expires_ts": number; +} + +export enum GroupCallState { + LocalCallFeedUninitialized = "local_call_feed_uninitialized", + InitializingLocalCallFeed = "initializing_local_call_feed", + LocalCallFeedInitialized = "local_call_feed_initialized", + Entering = "entering", + Entered = "entered", + Ended = "ended", +} + +interface ICallHandlers { + onCallFeedsChanged: (feeds: CallFeed[]) => void; + onCallStateChanged: (state: CallState, oldState: CallState | undefined) => void; + onCallHangup: (call: MatrixCall) => void; + onCallReplaced: (newCall: MatrixCall) => void; +} + +const CALL_MEMBER_STATE_TIMEOUT = 1000 * 60 * 60; // 1 hour + +const callMemberStateIsExpired = (event: MatrixEvent): boolean => { + const now = Date.now(); + const content = event?.getContent() ?? {}; + const expiresAt = typeof content["m.expires_ts"] === "number" ? content["m.expires_ts"] : -Infinity; + return expiresAt <= now; +}; + +function getCallUserId(call: MatrixCall): string | null { + return call.getOpponentMember()?.userId || call.invitee || null; +} + +export class GroupCall extends TypedEventEmitter< + GroupCallEvent | CallEvent, + GroupCallEventHandlerMap & CallEventHandlerMap +> { + // Config + public activeSpeakerInterval = 1000; + public retryCallInterval = 5000; + public participantTimeout = 1000 * 15; + public pttMaxTransmitTime = 1000 * 20; + + public state = GroupCallState.LocalCallFeedUninitialized; + public activeSpeaker?: string; // userId + public localCallFeed?: CallFeed; + public localScreenshareFeed?: CallFeed; + public localDesktopCapturerSourceId?: string; + public calls: MatrixCall[] = []; + public participants: RoomMember[] = []; + public userMediaFeeds: CallFeed[] = []; + public screenshareFeeds: CallFeed[] = []; + public groupCallId: string; + + private callHandlers: Map = new Map(); + private activeSpeakerLoopTimeout?: ReturnType; + private retryCallLoopTimeout?: ReturnType; + private retryCallCounts: Map = new Map(); + private reEmitter: ReEmitter; + private transmitTimer: ReturnType | null = null; + private memberStateExpirationTimers: Map> = new Map(); + private resendMemberStateTimer: ReturnType | null = null; + private initWithAudioMuted = false; + private initWithVideoMuted = false; + + constructor( + private client: MatrixClient, + public room: Room, + public type: GroupCallType, + public isPtt: boolean, + public intent: GroupCallIntent, + groupCallId?: string, + private dataChannelsEnabled?: boolean, + private dataChannelOptions?: IGroupCallDataChannelOptions, + ) { + super(); + this.reEmitter = new ReEmitter(this); + this.groupCallId = groupCallId || genCallID(); + + for (const stateEvent of this.getMemberStateEvents()) { + this.onMemberStateChanged(stateEvent); + } + } + + public async create() { + this.client.groupCallEventHandler!.groupCalls.set(this.room.roomId, this); + + await this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallPrefix, + { + "m.intent": this.intent, + "m.type": this.type, + "io.element.ptt": this.isPtt, + // TODO: Specify datachannels + "dataChannelsEnabled": this.dataChannelsEnabled, + "dataChannelOptions": this.dataChannelOptions, + }, + this.groupCallId, + ); + + return this; + } + + private setState(newState: GroupCallState): void { + const oldState = this.state; + this.state = newState; + this.emit(GroupCallEvent.GroupCallStateChanged, newState, oldState); + } + + public getLocalFeeds(): CallFeed[] { + const feeds: CallFeed[] = []; + + if (this.localCallFeed) feeds.push(this.localCallFeed); + if (this.localScreenshareFeed) feeds.push(this.localScreenshareFeed); + + return feeds; + } + + public hasLocalParticipant(): boolean { + const userId = this.client.getUserId(); + return this.participants.some((member) => member.userId === userId); + } + + public async initLocalCallFeed(): Promise { + logger.log(`groupCall ${this.groupCallId} initLocalCallFeed`); + + if (this.state !== GroupCallState.LocalCallFeedUninitialized) { + throw new Error(`Cannot initialize local call feed in the "${this.state}" state.`); + } + + this.setState(GroupCallState.InitializingLocalCallFeed); + + let stream: MediaStream; + + let disposed = false; + const onState = (state: GroupCallState) => { + if (state === GroupCallState.LocalCallFeedUninitialized) { + disposed = true; + } + }; + this.on(GroupCallEvent.GroupCallStateChanged, onState); + + try { + stream = await this.client.getMediaHandler().getUserMediaStream(true, this.type === GroupCallType.Video); + } catch (error) { + this.setState(GroupCallState.LocalCallFeedUninitialized); + throw error; + } finally { + this.off(GroupCallEvent.GroupCallStateChanged, onState); + } + + // The call could've been disposed while we were waiting + if (disposed) throw new Error("Group call disposed"); + + const userId = this.client.getUserId()!; + + const callFeed = new CallFeed({ + client: this.client, + roomId: this.room.roomId, + userId, + stream, + purpose: SDPStreamMetadataPurpose.Usermedia, + audioMuted: this.initWithAudioMuted || stream.getAudioTracks().length === 0 || this.isPtt, + videoMuted: this.initWithVideoMuted || stream.getVideoTracks().length === 0, + }); + + setTracksEnabled(stream.getAudioTracks(), !callFeed.isAudioMuted()); + setTracksEnabled(stream.getVideoTracks(), !callFeed.isVideoMuted()); + + this.localCallFeed = callFeed; + this.addUserMediaFeed(callFeed); + + this.setState(GroupCallState.LocalCallFeedInitialized); + + return callFeed; + } + + public async updateLocalUsermediaStream(stream: MediaStream) { + if (this.localCallFeed) { + const oldStream = this.localCallFeed.stream; + this.localCallFeed.setNewStream(stream); + const micShouldBeMuted = this.localCallFeed.isAudioMuted(); + const vidShouldBeMuted = this.localCallFeed.isVideoMuted(); + logger.log(`groupCall ${this.groupCallId} updateLocalUsermediaStream oldStream ${ + oldStream.id} newStream ${stream.id} micShouldBeMuted ${ + micShouldBeMuted} vidShouldBeMuted ${vidShouldBeMuted}`); + setTracksEnabled(stream.getAudioTracks(), !micShouldBeMuted); + setTracksEnabled(stream.getVideoTracks(), !vidShouldBeMuted); + this.client.getMediaHandler().stopUserMediaStream(oldStream); + } + } + + public async enter() { + if (!(this.state === GroupCallState.LocalCallFeedUninitialized || + this.state === GroupCallState.LocalCallFeedInitialized)) { + throw new Error(`Cannot enter call in the "${this.state}" state`); + } + + if (this.state === GroupCallState.LocalCallFeedUninitialized) { + await this.initLocalCallFeed(); + } + + this.addParticipant(this.room.getMember(this.client.getUserId()!)!); + + await this.sendMemberStateEvent(); + + this.activeSpeaker = undefined; + + this.setState(GroupCallState.Entered); + + logger.log(`Entered group call ${this.groupCallId}`); + + this.client.on(CallEventHandlerEvent.Incoming, this.onIncomingCall); + + const calls = this.client.callEventHandler!.calls.values(); + + for (const call of calls) { + this.onIncomingCall(call); + } + + // Set up participants for the members currently in the room. + // Other members will be picked up by the RoomState.members event. + for (const stateEvent of this.getMemberStateEvents()) { + this.onMemberStateChanged(stateEvent); + } + + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); + + this.onActiveSpeakerLoop(); + } + + private dispose() { + if (this.localCallFeed) { + this.removeUserMediaFeed(this.localCallFeed); + this.localCallFeed = undefined; + } + + if (this.localScreenshareFeed) { + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed.stream); + this.removeScreenshareFeed(this.localScreenshareFeed); + this.localScreenshareFeed = undefined; + this.localDesktopCapturerSourceId = undefined; + } + + this.client.getMediaHandler().stopAllStreams(); + + if (this.state !== GroupCallState.Entered) { + return; + } + + this.removeParticipant(this.room.getMember(this.client.getUserId()!)!); + + this.removeMemberStateEvent(); + + while (this.calls.length > 0) { + this.removeCall(this.calls[this.calls.length - 1], CallErrorCode.UserHangup); + } + + this.activeSpeaker = undefined; + clearTimeout(this.activeSpeakerLoopTimeout); + + this.retryCallCounts.clear(); + clearTimeout(this.retryCallLoopTimeout); + + for (const [userId] of this.memberStateExpirationTimers) { + clearTimeout(this.memberStateExpirationTimers.get(userId)); + this.memberStateExpirationTimers.delete(userId); + } + + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + + this.client.removeListener(CallEventHandlerEvent.Incoming, this.onIncomingCall); + } + + public leave() { + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + + this.dispose(); + this.setState(GroupCallState.LocalCallFeedUninitialized); + } + + public async terminate(emitStateEvent = true) { + this.dispose(); + + if (this.transmitTimer !== null) { + clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + + this.participants = []; + this.client.groupCallEventHandler!.groupCalls.delete(this.room.roomId); + + if (emitStateEvent) { + const existingStateEvent = this.room.currentState.getStateEvents( + EventType.GroupCallPrefix, this.groupCallId, + ); + + await this.client.sendStateEvent( + this.room.roomId, + EventType.GroupCallPrefix, + { + ...existingStateEvent.getContent(), + ["m.terminated"]: GroupCallTerminationReason.CallEnded, + }, + this.groupCallId, + ); + } + + this.client.emit(GroupCallEventHandlerEvent.Ended, this); + this.setState(GroupCallState.Ended); + } + + /** + * Local Usermedia + */ + + public isLocalVideoMuted() { + if (this.localCallFeed) { + return this.localCallFeed.isVideoMuted(); + } + + return true; + } + + public isMicrophoneMuted() { + if (this.localCallFeed) { + return this.localCallFeed.isAudioMuted(); + } + + return true; + } + + /** + * Sets the mute state of the local participants's microphone. + * @param {boolean} muted Whether to mute the microphone + * @returns {Promise} Whether muting/unmuting was successful + */ + public async setMicrophoneMuted(muted: boolean): Promise { + // hasAudioDevice can block indefinitely if the window has lost focus, + // and it doesn't make much sense to keep a device from being muted, so + // we always allow muted = true changes to go through + if (!muted && !await this.client.getMediaHandler().hasAudioDevice()) { + return false; + } + + const sendUpdatesBefore = !muted && this.isPtt; + + // set a timer for the maximum transmit time on PTT calls + if (this.isPtt) { + // Set or clear the max transmit timer + if (!muted && this.isMicrophoneMuted()) { + this.transmitTimer = setTimeout(() => { + this.setMicrophoneMuted(true); + }, this.pttMaxTransmitTime); + } else if (muted && !this.isMicrophoneMuted()) { + if (this.transmitTimer !== null) clearTimeout(this.transmitTimer); + this.transmitTimer = null; + } + } + + for (const call of this.calls) { + call.localUsermediaFeed?.setAudioVideoMuted(muted, null); + } + + if (sendUpdatesBefore) { + try { + await Promise.all(this.calls.map(c => c.sendMetadataUpdate())); + } catch (e) { + logger.info("Failed to send one or more metadata updates", e); + } + } + + if (this.localCallFeed) { + logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted stream ${ + this.localCallFeed.stream.id} muted ${muted}`); + this.localCallFeed.setAudioVideoMuted(muted, null); + // I don't believe its actually necessary to enable these tracks: they + // are the one on the groupcall's own CallFeed and are cloned before being + // given to any of the actual calls, so these tracks don't actually go + // anywhere. Let's do it anyway to avoid confusion. + setTracksEnabled(this.localCallFeed.stream.getAudioTracks(), !muted); + } else { + logger.log(`groupCall ${this.groupCallId} setMicrophoneMuted no stream muted ${muted}`); + this.initWithAudioMuted = muted; + } + + for (const call of this.calls) { + setTracksEnabled(call.localUsermediaFeed!.stream.getAudioTracks(), !muted); + } + + this.emit(GroupCallEvent.LocalMuteStateChanged, muted, this.isLocalVideoMuted()); + + if (!sendUpdatesBefore) { + try { + await Promise.all(this.calls.map(c => c.sendMetadataUpdate())); + } catch (e) { + logger.info("Failed to send one or more metadata updates", e); + } + } + + return true; + } + + /** + * Sets the mute state of the local participants's video. + * @param {boolean} muted Whether to mute the video + * @returns {Promise} Whether muting/unmuting was successful + */ + public async setLocalVideoMuted(muted: boolean): Promise { + // hasAudioDevice can block indefinitely if the window has lost focus, + // and it doesn't make much sense to keep a device from being muted, so + // we always allow muted = true changes to go through + if (!muted && !await this.client.getMediaHandler().hasVideoDevice()) { + return false; + } + + if (this.localCallFeed) { + logger.log(`groupCall ${this.groupCallId} setLocalVideoMuted stream ${ + this.localCallFeed.stream.id} muted ${muted}`); + this.localCallFeed.setAudioVideoMuted(null, muted); + setTracksEnabled(this.localCallFeed.stream.getVideoTracks(), !muted); + } else { + logger.log(`groupCall ${this.groupCallId} setLocalVideoMuted no stream muted ${muted}`); + this.initWithVideoMuted = muted; + } + + for (const call of this.calls) { + call.setLocalVideoMuted(muted); + } + + this.emit(GroupCallEvent.LocalMuteStateChanged, this.isMicrophoneMuted(), muted); + return true; + } + + public async setScreensharingEnabled( + enabled: boolean, opts: IScreensharingOpts = {}, + ): Promise { + if (enabled === this.isScreensharing()) { + return enabled; + } + + if (enabled) { + try { + logger.log("Asking for screensharing permissions..."); + const stream = await this.client.getMediaHandler().getScreensharingStream(opts); + + for (const track of stream.getTracks()) { + const onTrackEnded = () => { + this.setScreensharingEnabled(false); + track.removeEventListener("ended", onTrackEnded); + }; + + track.addEventListener("ended", onTrackEnded); + } + + logger.log("Screensharing permissions granted. Setting screensharing enabled on all calls"); + + this.localDesktopCapturerSourceId = opts.desktopCapturerSourceId; + this.localScreenshareFeed = new CallFeed({ + client: this.client, + roomId: this.room.roomId, + userId: this.client.getUserId()!, + stream, + purpose: SDPStreamMetadataPurpose.Screenshare, + audioMuted: false, + videoMuted: false, + }); + this.addScreenshareFeed(this.localScreenshareFeed); + + this.emit( + GroupCallEvent.LocalScreenshareStateChanged, + true, + this.localScreenshareFeed, + this.localDesktopCapturerSourceId, + ); + + // TODO: handle errors + await Promise.all(this.calls.map(call => call.pushLocalFeed( + this.localScreenshareFeed!.clone(), + ))); + + await this.sendMemberStateEvent(); + + return true; + } catch (error) { + if (opts.throwOnFail) throw error; + logger.error("Enabling screensharing error", error); + this.emit(GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.NoUserMedia, + "Failed to get screen-sharing stream: ", error as Error, + ), + ); + return false; + } + } else { + await Promise.all(this.calls.map(call => { + if (call.localScreensharingFeed) call.removeLocalFeed(call.localScreensharingFeed); + })); + this.client.getMediaHandler().stopScreensharingStream(this.localScreenshareFeed!.stream); + this.removeScreenshareFeed(this.localScreenshareFeed!); + this.localScreenshareFeed = undefined; + this.localDesktopCapturerSourceId = undefined; + await this.sendMemberStateEvent(); + this.emit(GroupCallEvent.LocalScreenshareStateChanged, false, undefined, undefined); + return false; + } + } + + public isScreensharing(): boolean { + return !!this.localScreenshareFeed; + } + + /** + * Call Setup + * + * There are two different paths for calls to be created: + * 1. Incoming calls triggered by the Call.incoming event. + * 2. Outgoing calls to the initial members of a room or new members + * as they are observed by the RoomState.members event. + */ + + private onIncomingCall = (newCall: MatrixCall) => { + // The incoming calls may be for another room, which we will ignore. + if (newCall.roomId !== this.room.roomId) { + return; + } + + if (newCall.state !== CallState.Ringing) { + logger.warn("Incoming call no longer in ringing state. Ignoring."); + return; + } + + if (!newCall.groupCallId || newCall.groupCallId !== this.groupCallId) { + logger.log(`Incoming call with groupCallId ${ + newCall.groupCallId} ignored because it doesn't match the current group call`); + newCall.reject(); + return; + } + + const opponentMemberId = newCall.getOpponentMember()?.userId; + const existingCall = opponentMemberId ? this.getCallByUserId(opponentMemberId) : null; + + if (existingCall && existingCall.callId === newCall.callId) { + return; + } + + logger.log(`GroupCall: incoming call from: ${opponentMemberId}`); + + // we are handlng this call as a PTT call, so enable PTT semantics + newCall.isPtt = this.isPtt; + + // Check if the user calling has an existing call and use this call instead. + if (existingCall) { + this.replaceCall(existingCall, newCall); + } else { + this.addCall(newCall); + } + + newCall.answerWithCallFeeds(this.getLocalFeeds().map((feed) => feed.clone())); + }; + + /** + * Room Member State + */ + + private getMemberStateEvents(): MatrixEvent[]; + private getMemberStateEvents(userId: string): MatrixEvent | null; + private getMemberStateEvents(userId?: string): MatrixEvent[] | MatrixEvent | null { + if (userId != null) { + const event = this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix, userId); + return callMemberStateIsExpired(event) ? null : event; + } else { + return this.room.currentState.getStateEvents(EventType.GroupCallMemberPrefix) + .filter(event => !callMemberStateIsExpired(event)); + } + } + + private async sendMemberStateEvent(): Promise { + const send = () => this.updateMemberCallState({ + "m.call_id": this.groupCallId, + "m.devices": [ + { + "device_id": this.client.getDeviceId()!, + "session_id": this.client.getSessionId(), + "feeds": this.getLocalFeeds().map((feed) => ({ + purpose: feed.purpose, + })), + // TODO: Add data channels + }, + ], + // TODO "m.foci" + }); + + const res = await send(); + + // Clear the old interval first, so that it isn't forgot + if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); + // Resend the state event every so often so it doesn't become stale + this.resendMemberStateTimer = setInterval(async () => { + logger.log("Resending call member state"); + await send(); + }, CALL_MEMBER_STATE_TIMEOUT * 3 / 4); + + return res; + } + + private async removeMemberStateEvent(): Promise { + if (this.resendMemberStateTimer !== null) clearInterval(this.resendMemberStateTimer); + this.resendMemberStateTimer = null; + return await this.updateMemberCallState(undefined, true); + } + + private async updateMemberCallState( + memberCallState?: IGroupCallRoomMemberCallState, + keepAlive = false, + ): Promise { + const localUserId = this.client.getUserId()!; + + const memberState = this.getMemberStateEvents(localUserId)?.getContent(); + + let calls: IGroupCallRoomMemberCallState[] = []; + + // Sanitize existing member state event + if (memberState && Array.isArray(memberState["m.calls"])) { + calls = memberState["m.calls"].filter((call) => !!call); + } + + const existingCallIndex = calls.findIndex((call) => call && call["m.call_id"] === this.groupCallId); + + if (existingCallIndex !== -1) { + if (memberCallState) { + calls.splice(existingCallIndex, 1, memberCallState); + } else { + calls.splice(existingCallIndex, 1); + } + } else if (memberCallState) { + calls.push(memberCallState); + } + + const content = { + "m.calls": calls, + "m.expires_ts": Date.now() + CALL_MEMBER_STATE_TIMEOUT, + }; + + return this.client.sendStateEvent( + this.room.roomId, EventType.GroupCallMemberPrefix, content, localUserId, { keepAlive }, + ); + } + + public onMemberStateChanged = async (event: MatrixEvent) => { + // If we haven't entered the call yet, we don't care + if (this.state !== GroupCallState.Entered) { + return; + } + + // The member events may be received for another room, which we will ignore. + if (event.getRoomId() !== this.room.roomId) return; + + const member = this.room.getMember(event.getStateKey()!); + if (!member) { + logger.warn(`Couldn't find room member for ${event.getStateKey()}: ignoring member state event!`); + return; + } + + // Don't process your own member. + const localUserId = this.client.getUserId()!; + if (member.userId === localUserId) return; + + logger.debug(`Processing member state event for ${member.userId}`); + + const ignore = () => { + this.removeParticipant(member); + clearTimeout(this.memberStateExpirationTimers.get(member.userId)); + this.memberStateExpirationTimers.delete(member.userId); + }; + + const content = event.getContent(); + const callsState = !callMemberStateIsExpired(event) && Array.isArray(content["m.calls"]) + ? content["m.calls"].filter((call) => call) + : []; // Ignore expired device data + + if (callsState.length === 0) { + logger.info(`Ignoring member state from ${member.userId} member not in any calls.`); + ignore(); + return; + } + + // Currently we only support a single call per room. So grab the first call. + const callState = callsState[0]; + const callId = callState["m.call_id"]; + + if (!callId) { + logger.warn(`Room member ${member.userId} does not have a valid m.call_id set. Ignoring.`); + ignore(); + return; + } + + if (callId !== this.groupCallId) { + logger.warn(`Call id ${callId} does not match group call id ${this.groupCallId}, ignoring.`); + ignore(); + return; + } + + this.addParticipant(member); + + clearTimeout(this.memberStateExpirationTimers.get(member.userId)); + this.memberStateExpirationTimers.set(member.userId, setTimeout(() => { + logger.warn(`Call member state for ${member.userId} has expired`); + this.removeParticipant(member); + }, content["m.expires_ts"] - Date.now())); + + // Only initiate a call with a user who has a userId that is lexicographically + // less than your own. Otherwise, that user will call you. + if (member.userId < localUserId) { + logger.debug(`Waiting for ${member.userId} to send call invite.`); + return; + } + + const opponentDevice = this.getDeviceForMember(member.userId); + + if (!opponentDevice) { + logger.warn(`No opponent device found for ${member.userId}, ignoring.`); + this.emit( + GroupCallEvent.Error, + new GroupCallUnknownDeviceError(member.userId), + ); + return; + } + + const existingCall = this.getCallByUserId(member.userId); + + if ( + existingCall && + existingCall.getOpponentSessionId() === opponentDevice.session_id + ) { + return; + } + + const newCall = createNewMatrixCall( + this.client, + this.room.roomId, + { + invitee: member.userId, + opponentDeviceId: opponentDevice.device_id, + opponentSessionId: opponentDevice.session_id, + groupCallId: this.groupCallId, + }, + ); + + if (!newCall) { + logger.error("Failed to create call!"); + return; + } + + if (existingCall) { + logger.debug(`Replacing call ${existingCall.callId} to ${member.userId} with ${newCall.callId}`); + this.replaceCall(existingCall, newCall, CallErrorCode.NewSession); + } else { + logger.debug(`Adding call ${newCall.callId} to ${member.userId}`); + this.addCall(newCall); + } + + newCall.isPtt = this.isPtt; + + const requestScreenshareFeed = opponentDevice.feeds.some( + (feed) => feed.purpose === SDPStreamMetadataPurpose.Screenshare); + + logger.debug( + `Placing call to ${member.userId}/${opponentDevice.device_id} session ID ${opponentDevice.session_id}.`, + ); + + try { + await newCall.placeCallWithCallFeeds( + this.getLocalFeeds().map(feed => feed.clone()), + requestScreenshareFeed, + ); + } catch (e) { + logger.warn(`Failed to place call to ${member.userId}!`, e); + if (e instanceof CallError && e.code === GroupCallErrorCode.UnknownDevice) { + this.emit(GroupCallEvent.Error, e); + } else { + this.emit( + GroupCallEvent.Error, + new GroupCallError( + GroupCallErrorCode.PlaceCallFailed, + `Failed to place call to ${member.userId}.`, + ), + ); + } + this.removeCall(newCall, CallErrorCode.SignallingFailed); + return; + } + + if (this.dataChannelsEnabled) { + newCall.createDataChannel("datachannel", this.dataChannelOptions); + } + }; + + public getDeviceForMember(userId: string): IGroupCallRoomMemberDevice | undefined { + const memberStateEvent = this.getMemberStateEvents(userId); + + if (!memberStateEvent) { + return undefined; + } + + const memberState = memberStateEvent.getContent(); + const memberGroupCallState = memberState["m.calls"]?.find( + (call) => call && call["m.call_id"] === this.groupCallId); + + if (!memberGroupCallState) { + return undefined; + } + + const memberDevices = memberGroupCallState["m.devices"]; + + if (!memberDevices || memberDevices.length === 0) { + return undefined; + } + + // NOTE: For now we only support one device so we use the device id in the first source. + return memberDevices[0]; + } + + private onRetryCallLoop = () => { + for (const event of this.getMemberStateEvents()) { + const memberId = event.getStateKey()!; + const existingCall = this.calls.find((call) => getCallUserId(call) === memberId); + const retryCallCount = this.retryCallCounts.get(memberId) || 0; + + if (!existingCall && retryCallCount < 3) { + this.retryCallCounts.set(memberId, retryCallCount + 1); + this.onMemberStateChanged(event); + } + } + + this.retryCallLoopTimeout = setTimeout(this.onRetryCallLoop, this.retryCallInterval); + }; + + /** + * Call Event Handlers + */ + + public getCallByUserId(userId: string): MatrixCall | undefined { + return this.calls.find((call) => getCallUserId(call) === userId); + } + + private addCall(call: MatrixCall) { + this.calls.push(call); + this.initCall(call); + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + + private replaceCall(existingCall: MatrixCall, replacementCall: MatrixCall, hangupReason = CallErrorCode.Replaced) { + const existingCallIndex = this.calls.indexOf(existingCall); + + if (existingCallIndex === -1) { + throw new Error("Couldn't find call to replace"); + } + + this.calls.splice(existingCallIndex, 1, replacementCall); + + this.disposeCall(existingCall, hangupReason); + this.initCall(replacementCall); + + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + + private removeCall(call: MatrixCall, hangupReason: CallErrorCode) { + this.disposeCall(call, hangupReason); + + const callIndex = this.calls.indexOf(call); + + if (callIndex === -1) { + throw new Error("Couldn't find call to remove"); + } + + this.calls.splice(callIndex, 1); + + this.emit(GroupCallEvent.CallsChanged, this.calls); + } + + private initCall(call: MatrixCall) { + const opponentMemberId = getCallUserId(call); + + if (!opponentMemberId) { + throw new Error("Cannot init call without user id"); + } + + const onCallFeedsChanged = () => this.onCallFeedsChanged(call); + const onCallStateChanged = + (state: CallState, oldState: CallState | undefined) => this.onCallStateChanged(call, state, oldState); + const onCallHangup = this.onCallHangup; + const onCallReplaced = (newCall: MatrixCall) => this.replaceCall(call, newCall); + + this.callHandlers.set(opponentMemberId, { + onCallFeedsChanged, + onCallStateChanged, + onCallHangup, + onCallReplaced, + }); + + call.on(CallEvent.FeedsChanged, onCallFeedsChanged); + call.on(CallEvent.State, onCallStateChanged); + call.on(CallEvent.Hangup, onCallHangup); + call.on(CallEvent.Replaced, onCallReplaced); + + this.reEmitter.reEmit(call, Object.values(CallEvent)); + + onCallFeedsChanged(); + } + + private disposeCall(call: MatrixCall, hangupReason: CallErrorCode) { + const opponentMemberId = getCallUserId(call); + + if (!opponentMemberId) { + throw new Error("Cannot dispose call without user id"); + } + + const { + onCallFeedsChanged, + onCallStateChanged, + onCallHangup, + onCallReplaced, + } = this.callHandlers.get(opponentMemberId)!; + + call.removeListener(CallEvent.FeedsChanged, onCallFeedsChanged); + call.removeListener(CallEvent.State, onCallStateChanged); + call.removeListener(CallEvent.Hangup, onCallHangup); + call.removeListener(CallEvent.Replaced, onCallReplaced); + + this.callHandlers.delete(opponentMemberId); + + if (call.hangupReason === CallErrorCode.Replaced) { + return; + } + + if (call.state !== CallState.Ended) { + call.hangup(hangupReason, false); + } + + const usermediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + + if (usermediaFeed) { + this.removeUserMediaFeed(usermediaFeed); + } + + const screenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); + + if (screenshareFeed) { + this.removeScreenshareFeed(screenshareFeed); + } + } + + private onCallFeedsChanged = (call: MatrixCall) => { + const opponentMemberId = getCallUserId(call); + + if (!opponentMemberId) { + throw new Error("Cannot change call feeds without user id"); + } + + const currentUserMediaFeed = this.getUserMediaFeedByUserId(opponentMemberId); + const remoteUsermediaFeed = call.remoteUsermediaFeed; + const remoteFeedChanged = remoteUsermediaFeed !== currentUserMediaFeed; + + if (remoteFeedChanged) { + if (!currentUserMediaFeed && remoteUsermediaFeed) { + this.addUserMediaFeed(remoteUsermediaFeed); + } else if (currentUserMediaFeed && remoteUsermediaFeed) { + this.replaceUserMediaFeed(currentUserMediaFeed, remoteUsermediaFeed); + } else if (currentUserMediaFeed && !remoteUsermediaFeed) { + this.removeUserMediaFeed(currentUserMediaFeed); + } + } + + const currentScreenshareFeed = this.getScreenshareFeedByUserId(opponentMemberId); + const remoteScreensharingFeed = call.remoteScreensharingFeed; + const remoteScreenshareFeedChanged = remoteScreensharingFeed !== currentScreenshareFeed; + + if (remoteScreenshareFeedChanged) { + if (!currentScreenshareFeed && remoteScreensharingFeed) { + this.addScreenshareFeed(remoteScreensharingFeed); + } else if (currentScreenshareFeed && remoteScreensharingFeed) { + this.replaceScreenshareFeed(currentScreenshareFeed, remoteScreensharingFeed); + } else if (currentScreenshareFeed && !remoteScreensharingFeed) { + this.removeScreenshareFeed(currentScreenshareFeed); + } + } + }; + + private onCallStateChanged = (call: MatrixCall, state: CallState, _oldState: CallState | undefined) => { + const audioMuted = this.localCallFeed!.isAudioMuted(); + + if ( + call.localUsermediaStream && + call.isMicrophoneMuted() !== audioMuted + ) { + call.setMicrophoneMuted(audioMuted); + } + + const videoMuted = this.localCallFeed!.isVideoMuted(); + + if ( + call.localUsermediaStream && + call.isLocalVideoMuted() !== videoMuted + ) { + call.setLocalVideoMuted(videoMuted); + } + + if (state === CallState.Connected) { + this.retryCallCounts.delete(getCallUserId(call)!); + } + }; + + private onCallHangup = (call: MatrixCall) => { + if (call.hangupReason === CallErrorCode.Replaced) { + return; + } + + this.removeCall(call, call.hangupReason as CallErrorCode); + }; + + /** + * UserMedia CallFeed Event Handlers + */ + + public getUserMediaFeedByUserId(userId: string) { + return this.userMediaFeeds.find((feed) => feed.userId === userId); + } + + private addUserMediaFeed(callFeed: CallFeed) { + this.userMediaFeeds.push(callFeed); + callFeed.measureVolumeActivity(true); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + } + + private replaceUserMediaFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { + const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === existingFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find user media feed to replace"); + } + + this.userMediaFeeds.splice(feedIndex, 1, replacementFeed); + + existingFeed.dispose(); + replacementFeed.measureVolumeActivity(true); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + } + + private removeUserMediaFeed(callFeed: CallFeed) { + const feedIndex = this.userMediaFeeds.findIndex((feed) => feed.userId === callFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find user media feed to remove"); + } + + this.userMediaFeeds.splice(feedIndex, 1); + + callFeed.dispose(); + this.emit(GroupCallEvent.UserMediaFeedsChanged, this.userMediaFeeds); + + if ( + this.activeSpeaker === callFeed.userId && + this.userMediaFeeds.length > 0 + ) { + this.activeSpeaker = this.userMediaFeeds[0].userId; + this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); + } + } + + private onActiveSpeakerLoop = () => { + let topAvg: number | undefined = undefined; + let nextActiveSpeaker: string | undefined = undefined; + + for (const callFeed of this.userMediaFeeds) { + if (callFeed.userId === this.client.getUserId() && this.userMediaFeeds.length > 1) { + continue; + } + + let total = 0; + + for (let i = 0; i < callFeed.speakingVolumeSamples.length; i++) { + const volume = callFeed.speakingVolumeSamples[i]; + total += Math.max(volume, SPEAKING_THRESHOLD); + } + + const avg = total / callFeed.speakingVolumeSamples.length; + + if (!topAvg || avg > topAvg) { + topAvg = avg; + nextActiveSpeaker = callFeed.userId; + } + } + + if (nextActiveSpeaker && this.activeSpeaker !== nextActiveSpeaker && topAvg && topAvg > SPEAKING_THRESHOLD) { + this.activeSpeaker = nextActiveSpeaker; + this.emit(GroupCallEvent.ActiveSpeakerChanged, this.activeSpeaker); + } + + this.activeSpeakerLoopTimeout = setTimeout( + this.onActiveSpeakerLoop, + this.activeSpeakerInterval, + ); + }; + + /** + * Screenshare Call Feed Event Handlers + */ + + public getScreenshareFeedByUserId(userId: string) { + return this.screenshareFeeds.find((feed) => feed.userId === userId); + } + + private addScreenshareFeed(callFeed: CallFeed) { + this.screenshareFeeds.push(callFeed); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } + + private replaceScreenshareFeed(existingFeed: CallFeed, replacementFeed: CallFeed) { + const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === existingFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find screenshare feed to replace"); + } + + this.screenshareFeeds.splice(feedIndex, 1, replacementFeed); + + existingFeed.dispose(); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } + + private removeScreenshareFeed(callFeed: CallFeed) { + const feedIndex = this.screenshareFeeds.findIndex((feed) => feed.userId === callFeed.userId); + + if (feedIndex === -1) { + throw new Error("Couldn't find screenshare feed to remove"); + } + + this.screenshareFeeds.splice(feedIndex, 1); + + callFeed.dispose(); + this.emit(GroupCallEvent.ScreenshareFeedsChanged, this.screenshareFeeds); + } + + /** + * Participant Management + */ + + private addParticipant(member: RoomMember) { + if (this.participants.find((m) => m.userId === member.userId)) { + return; + } + + this.participants.push(member); + + this.emit(GroupCallEvent.ParticipantsChanged, this.participants); + this.client.emit(GroupCallEventHandlerEvent.Participants, this.participants, this); + } + + private removeParticipant(member: RoomMember) { + const index = this.participants.findIndex((m) => m.userId === member.userId); + + if (index === -1) { + return; + } + + this.participants.splice(index, 1); + + this.emit(GroupCallEvent.ParticipantsChanged, this.participants); + this.client.emit(GroupCallEventHandlerEvent.Participants, this.participants, this); + } +} diff --git a/src/webrtc/groupCallEventHandler.ts b/src/webrtc/groupCallEventHandler.ts new file mode 100644 index 00000000000..86df722895d --- /dev/null +++ b/src/webrtc/groupCallEventHandler.ts @@ -0,0 +1,233 @@ +/* +Copyright 2021 Šimon Brandner + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixEvent } from '../models/event'; +import { MatrixClient, ClientEvent } from '../client'; +import { + GroupCall, + GroupCallIntent, + GroupCallType, + IGroupCallDataChannelOptions, +} from "./groupCall"; +import { Room } from "../models/room"; +import { RoomState, RoomStateEvent } from "../models/room-state"; +import { RoomMember } from "../models/room-member"; +import { logger } from '../logger'; +import { EventType } from "../@types/event"; +import { SyncState } from '../sync'; + +export enum GroupCallEventHandlerEvent { + Incoming = "GroupCall.incoming", + Ended = "GroupCall.ended", + Participants = "GroupCall.participants", +} + +export type GroupCallEventHandlerEventHandlerMap = { + [GroupCallEventHandlerEvent.Incoming]: (call: GroupCall) => void; + [GroupCallEventHandlerEvent.Ended]: (call: GroupCall) => void; + [GroupCallEventHandlerEvent.Participants]: (participants: RoomMember[], call: GroupCall) => void; +}; + +interface RoomDeferred { + prom: Promise; + resolve?: () => void; +} + +export class GroupCallEventHandler { + public groupCalls = new Map(); // roomId -> GroupCall + + // All rooms we know about and whether we've seen a 'Room' event + // for them. The promise will be fulfilled once we've processed that + // event which means we're "up to date" on what calls are in a room + // and get + private roomDeferreds = new Map(); + + constructor(private client: MatrixClient) { } + + public async start(): Promise { + // We wait until the client has started syncing for real. + // This is because we only support one call at a time, and want + // the latest. We therefore want the latest state of the room before + // we create a group call for the room so we can be fairly sure that + // the group call we create is really the latest one. + if (this.client.getSyncState() !== SyncState.Syncing) { + logger.debug("Waiting for client to start syncing..."); + await new Promise(resolve => { + const onSync = () => { + if (this.client.getSyncState() === SyncState.Syncing) { + this.client.off(ClientEvent.Sync, onSync); + return resolve(); + } + }; + this.client.on(ClientEvent.Sync, onSync); + }); + } + + const rooms = this.client.getRooms(); + + for (const room of rooms) { + this.createGroupCallForRoom(room); + } + + this.client.on(ClientEvent.Room, this.onRoomsChanged); + this.client.on(RoomStateEvent.Events, this.onRoomStateChanged); + } + + public stop(): void { + this.client.removeListener(RoomStateEvent.Events, this.onRoomStateChanged); + } + + private getRoomDeferred(roomId: string): RoomDeferred { + let deferred = this.roomDeferreds.get(roomId); + if (deferred === undefined) { + let resolveFunc: () => void; + deferred = { + prom: new Promise(resolve => { + resolveFunc = resolve; + }), + }; + deferred.resolve = resolveFunc!; + this.roomDeferreds.set(roomId, deferred); + } + + return deferred; + } + + public waitUntilRoomReadyForGroupCalls(roomId: string): Promise { + return this.getRoomDeferred(roomId).prom; + } + + public getGroupCallById(groupCallId: string): GroupCall | undefined { + return [...this.groupCalls.values()].find((groupCall) => groupCall.groupCallId === groupCallId); + } + + private createGroupCallForRoom(room: Room): void { + const callEvents = room.currentState.getStateEvents(EventType.GroupCallPrefix); + const sortedCallEvents = callEvents.sort((a, b) => b.getTs() - a.getTs()); + + for (const callEvent of sortedCallEvents) { + const content = callEvent.getContent(); + + if (content["m.terminated"]) { + continue; + } + + logger.debug( + `Choosing group call ${callEvent.getStateKey()} with TS ` + + `${callEvent.getTs()} for room ${room.roomId} from ${callEvents.length} possible calls.`, + ); + + this.createGroupCallFromRoomStateEvent(callEvent); + break; + } + + logger.info("Group call event handler processed room", room.roomId); + this.getRoomDeferred(room.roomId).resolve!(); + } + + private createGroupCallFromRoomStateEvent(event: MatrixEvent): GroupCall | undefined { + const roomId = event.getRoomId(); + const content = event.getContent(); + + const room = this.client.getRoom(roomId); + + if (!room) { + logger.warn(`Couldn't find room ${roomId} for GroupCall`); + return; + } + + const groupCallId = event.getStateKey(); + + const callType = content["m.type"]; + + if (!Object.values(GroupCallType).includes(callType)) { + logger.warn(`Received invalid group call type ${callType} for room ${roomId}.`); + return; + } + + const callIntent = content["m.intent"]; + + if (!Object.values(GroupCallIntent).includes(callIntent)) { + logger.warn(`Received invalid group call intent ${callType} for room ${roomId}.`); + return; + } + + const isPtt = Boolean(content["io.element.ptt"]); + + let dataChannelOptions: IGroupCallDataChannelOptions | undefined; + + if (content?.dataChannelsEnabled && content?.dataChannelOptions) { + // Pull out just the dataChannelOptions we want to support. + const { ordered, maxPacketLifeTime, maxRetransmits, protocol } = content.dataChannelOptions; + dataChannelOptions = { ordered, maxPacketLifeTime, maxRetransmits, protocol }; + } + + const groupCall = new GroupCall( + this.client, + room, + callType, + isPtt, + callIntent, + groupCallId, + content?.dataChannelsEnabled, + dataChannelOptions, + ); + + this.groupCalls.set(room.roomId, groupCall); + this.client.emit(GroupCallEventHandlerEvent.Incoming, groupCall); + + return groupCall; + } + + private onRoomsChanged = (room: Room) => { + this.createGroupCallForRoom(room); + }; + + private onRoomStateChanged = (event: MatrixEvent, state: RoomState): void => { + const eventType = event.getType(); + + if (eventType === EventType.GroupCallPrefix) { + const groupCallId = event.getStateKey(); + const content = event.getContent(); + + const currentGroupCall = this.groupCalls.get(state.roomId); + + if (!currentGroupCall && !content["m.terminated"]) { + this.createGroupCallFromRoomStateEvent(event); + } else if (currentGroupCall && currentGroupCall.groupCallId === groupCallId) { + if (content["m.terminated"]) { + currentGroupCall.terminate(false); + } else if (content["m.type"] !== currentGroupCall.type) { + // TODO: Handle the callType changing when the room state changes + logger.warn(`The group call type changed for room: ${ + state.roomId}. Changing the group call type is currently unsupported.`); + } + } else if (currentGroupCall && currentGroupCall.groupCallId !== groupCallId) { + // TODO: Handle new group calls and multiple group calls + logger.warn(`Multiple group calls detected for room: ${ + state.roomId}. Multiple group calls are currently unsupported.`); + } + } else if (eventType === EventType.GroupCallMemberPrefix) { + const groupCall = this.groupCalls.get(state.roomId); + + if (!groupCall) { + return; + } + + groupCall.onMemberStateChanged(event); + } + }; +} diff --git a/src/webrtc/mediaHandler.ts b/src/webrtc/mediaHandler.ts index 81d0aa74faa..3cea257caca 100644 --- a/src/webrtc/mediaHandler.ts +++ b/src/webrtc/mediaHandler.ts @@ -17,18 +17,46 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { TypedEventEmitter } from "../models/typed-event-emitter"; +import { GroupCallType, GroupCallState } from "../webrtc/groupCall"; import { logger } from "../logger"; import { MatrixClient } from "../client"; -import { CallState } from "./call"; -export class MediaHandler { +export enum MediaHandlerEvent { + LocalStreamsChanged = "local_streams_changed" +} + +export type MediaHandlerEventHandlerMap = { + [MediaHandlerEvent.LocalStreamsChanged]: () => void; +}; + +export interface IScreensharingOpts { + desktopCapturerSourceId?: string; + audio?: boolean; + // For electron screen capture, there are very few options for detecting electron + // apart from inspecting the user agent or just trying getDisplayMedia() and + // catching the failure, so we do the latter - this flag tells the function to just + // throw an error so we can catch it in this case, rather than logging and emitting. + throwOnFail?: boolean; +} + +export class MediaHandler extends TypedEventEmitter< + MediaHandlerEvent.LocalStreamsChanged, MediaHandlerEventHandlerMap +> { private audioInput?: string; private videoInput?: string; private localUserMediaStream?: MediaStream; public userMediaStreams: MediaStream[] = []; public screensharingStreams: MediaStream[] = []; - constructor(private client: MatrixClient) { } + constructor(private client: MatrixClient) { + super(); + } + + public restoreMediaSettings(audioInput: string, videoInput: string) { + this.audioInput = audioInput; + this.videoInput = videoInput; + } /** * Set an audio input device to use for MatrixCalls @@ -59,6 +87,19 @@ export class MediaHandler { } /** + * Set media input devices to use for MatrixCalls + * @param {string} audioInput the identifier for the audio device + * @param {string} videoInput the identifier for the video device + * undefined treated as unset + */ + public async setMediaInputs(audioInput: string, videoInput: string): Promise { + logger.log(`mediaHandler setMediaInputs audioInput: ${audioInput} videoInput: ${videoInput}`); + this.audioInput = audioInput; + this.videoInput = videoInput; + await this.updateLocalUsermediaStreams(); + } + + /* * Requests new usermedia streams and replace the old ones */ public async updateLocalUsermediaStreams(): Promise { @@ -72,16 +113,53 @@ export class MediaHandler { }); } + for (const stream of this.userMediaStreams) { + logger.log(`mediaHandler stopping all tracks for stream ${stream.id}`); + for (const track of stream.getTracks()) { + track.stop(); + } + } + + this.userMediaStreams = []; + this.localUserMediaStream = undefined; + for (const call of this.client.callEventHandler!.calls.values()) { - if (call.state === CallState.Ended || !callMediaStreamParams.has(call.callId)) continue; + if (call.callHasEnded() || !callMediaStreamParams.has(call.callId)) { + continue; + } const { audio, video } = callMediaStreamParams.get(call.callId)!; - // This stream won't be reusable as we will replace the tracks of the old stream - const stream = await this.getUserMediaStream(audio, video, false); + logger.log(`mediaHandler updateLocalUsermediaStreams getUserMediaStream call ${call.callId}`); + const stream = await this.getUserMediaStream(audio, video); + + if (call.callHasEnded()) { + continue; + } await call.updateLocalUsermediaStream(stream); } + + for (const groupCall of this.client.groupCallEventHandler!.groupCalls.values()) { + if (!groupCall.localCallFeed) { + continue; + } + + logger.log(`mediaHandler updateLocalUsermediaStreams getUserMediaStream groupCall ${ + groupCall.groupCallId}`); + const stream = await this.getUserMediaStream( + true, + groupCall.type === GroupCallType.Video, + ); + + if (groupCall.state === GroupCallState.Ended) { + continue; + } + + await groupCall.updateLocalUsermediaStream(stream); + } + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } public async hasAudioDevice(): Promise { @@ -106,16 +184,35 @@ export class MediaHandler { let stream: MediaStream; - if ( - !this.localUserMediaStream || - (this.localUserMediaStream.getAudioTracks().length === 0 && shouldRequestAudio) || - (this.localUserMediaStream.getVideoTracks().length === 0 && shouldRequestVideo) || - (this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput) || - (this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput) - ) { + let canReuseStream = true; + if (this.localUserMediaStream) { + // This code checks that the device ID is the same as the localUserMediaStream stream, but we update + // the localUserMediaStream whenever the device ID changes (apart from when restoring) so it's not + // clear why this would ever be different, unless there's a race. + if (shouldRequestAudio) { + if ( + this.localUserMediaStream.getAudioTracks().length === 0 || + this.localUserMediaStream.getAudioTracks()[0]?.getSettings()?.deviceId !== this.audioInput + ) { + canReuseStream = false; + } + } + if (shouldRequestVideo) { + if ( + this.localUserMediaStream.getVideoTracks().length === 0 || + this.localUserMediaStream.getVideoTracks()[0]?.getSettings()?.deviceId !== this.videoInput) { + canReuseStream = false; + } + } + } else { + canReuseStream = false; + } + + if (!canReuseStream) { const constraints = this.getUserMediaContraints(shouldRequestAudio, shouldRequestVideo); - logger.log("Getting user media with constraints", constraints); stream = await navigator.mediaDevices.getUserMedia(constraints); + logger.log(`mediaHandler getUserMediaStream streamId ${stream.id} shouldRequestAudio ${ + shouldRequestAudio} shouldRequestVideo ${shouldRequestVideo}`, constraints); for (const track of stream.getTracks()) { const settings = track.getSettings(); @@ -131,7 +228,9 @@ export class MediaHandler { this.localUserMediaStream = stream; } } else { - stream = this.localUserMediaStream.clone(); + stream = this.localUserMediaStream!.clone(); + logger.log(`mediaHandler clone userMediaStream ${this.localUserMediaStream?.id} new stream ${ + stream.id} shouldRequestAudio ${shouldRequestAudio} shouldRequestVideo ${shouldRequestVideo}`); if (!shouldRequestAudio) { for (const track of stream.getAudioTracks()) { @@ -150,6 +249,8 @@ export class MediaHandler { this.userMediaStreams.push(stream); } + this.emit(MediaHandlerEvent.LocalStreamsChanged); + return stream; } @@ -157,7 +258,7 @@ export class MediaHandler { * Stops all tracks on the provided usermedia stream */ public stopUserMediaStream(mediaStream: MediaStream) { - logger.debug("Stopping usermedia stream", mediaStream.id); + logger.log(`mediaHandler stopUserMediaStream stopping stream ${mediaStream.id}`); for (const track of mediaStream.getTracks()) { track.stop(); } @@ -169,6 +270,8 @@ export class MediaHandler { this.userMediaStreams.splice(index, 1); } + this.emit(MediaHandlerEvent.LocalStreamsChanged); + if (this.localUserMediaStream === mediaStream) { this.localUserMediaStream = undefined; } @@ -179,23 +282,19 @@ export class MediaHandler { * @param reusable is allowed to be reused by the MediaHandler * @returns {MediaStream} based on passed parameters */ - public async getScreensharingStream( - desktopCapturerSourceId?: string, - reusable = true, - ): Promise { + public async getScreensharingStream(opts: IScreensharingOpts = {}, reusable = true): Promise { let stream: MediaStream; if (this.screensharingStreams.length === 0) { - const screenshareConstraints = this.getScreenshareContraints(desktopCapturerSourceId); - if (!screenshareConstraints) return null; + const screenshareConstraints = this.getScreenshareContraints(opts); - if (desktopCapturerSourceId) { + if (opts.desktopCapturerSourceId) { // We are using Electron - logger.debug("Getting screensharing stream using getUserMedia()", desktopCapturerSourceId); + logger.debug("Getting screensharing stream using getUserMedia()", opts); stream = await navigator.mediaDevices.getUserMedia(screenshareConstraints); } else { // We are not using Electron - logger.debug("Getting screensharing stream using getDisplayMedia()"); + logger.debug("Getting screensharing stream using getDisplayMedia()", opts); stream = await navigator.mediaDevices.getDisplayMedia(screenshareConstraints); } } else { @@ -208,6 +307,8 @@ export class MediaHandler { this.screensharingStreams.push(stream); } + this.emit(MediaHandlerEvent.LocalStreamsChanged); + return stream; } @@ -226,6 +327,8 @@ export class MediaHandler { logger.debug("Splicing screensharing stream out stream array", mediaStream.id); this.screensharingStreams.splice(index, 1); } + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } /** @@ -233,6 +336,7 @@ export class MediaHandler { */ public stopAllStreams() { for (const stream of this.userMediaStreams) { + logger.log(`mediaHandler stopAllStreams stopping stream ${stream.id}`); for (const track of stream.getTracks()) { track.stop(); } @@ -247,6 +351,8 @@ export class MediaHandler { this.userMediaStreams = []; this.screensharingStreams = []; this.localUserMediaStream = undefined; + + this.emit(MediaHandlerEvent.LocalStreamsChanged); } private getUserMediaContraints(audio: boolean, video: boolean): MediaStreamConstraints { @@ -273,11 +379,12 @@ export class MediaHandler { }; } - private getScreenshareContraints(desktopCapturerSourceId?: string): DesktopCapturerConstraints { + private getScreenshareContraints(opts: IScreensharingOpts): DesktopCapturerConstraints { + const { desktopCapturerSourceId, audio } = opts; if (desktopCapturerSourceId) { logger.debug("Using desktop capturer source", desktopCapturerSourceId); return { - audio: false, + audio: audio ?? false, video: { mandatory: { chromeMediaSource: "desktop", @@ -288,7 +395,7 @@ export class MediaHandler { } else { logger.debug("Not using desktop capturer source"); return { - audio: false, + audio: audio ?? false, video: true, }; } diff --git a/yarn.lock b/yarn.lock index 6daa4bbc929..9f3a92b9b4a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1197,6 +1197,16 @@ slash "^3.0.0" strip-ansi "^6.0.0" +"@jest/environment@^28.1.3": + version "28.1.3" + resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-28.1.3.tgz#abed43a6b040a4c24fdcb69eab1f97589b2d663e" + integrity sha512-1bf40cMFTEkKyEf585R9Iz1WayDjHoHqvts0XFYEqyKM3cFWDpeMoqKKTAF9LSYQModPUlh8FKptoM2YcMWAXA== + dependencies: + "@jest/fake-timers" "^28.1.3" + "@jest/types" "^28.1.3" + "@types/node" "*" + jest-mock "^28.1.3" + "@jest/environment@^29.2.2": version "29.2.2" resolved "https://registry.yarnpkg.com/@jest/environment/-/environment-29.2.2.tgz#481e729048d42e87d04842c38aa4d09c507f53b0" @@ -1229,6 +1239,18 @@ expect "^29.2.2" jest-snapshot "^29.2.2" +"@jest/fake-timers@^28.1.3": + version "28.1.3" + resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-28.1.3.tgz#230255b3ad0a3d4978f1d06f70685baea91c640e" + integrity sha512-D/wOkL2POHv52h+ok5Oj/1gOG9HSywdoPtFsRCUmlCILXNn5eIWmcnd3DIiWlJnpGvQtmajqBP95Ei0EimxfLw== + dependencies: + "@jest/types" "^28.1.3" + "@sinonjs/fake-timers" "^9.1.2" + "@types/node" "*" + jest-message-util "^28.1.3" + jest-mock "^28.1.3" + jest-util "^28.1.3" + "@jest/fake-timers@^29.2.2": version "29.2.2" resolved "https://registry.yarnpkg.com/@jest/fake-timers/-/fake-timers-29.2.2.tgz#d8332e6e3cfa99cde4bc87d04a17d6b699deb340" @@ -1586,6 +1608,11 @@ dependencies: "@sinonjs/commons" "^1.7.0" +"@tootallnate/once@2": + version "2.0.0" + resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-2.0.0.tgz#f544a148d3ab35801c1f633a7441fd87c2e484bf" + integrity sha512-XCuKFP5PS55gnMVu3dty8KPatLqUoy/ZYzDzAGCQ8JNFCkLXzmI7vNHCR+XpbZaMWQK/vQubr7PkYq8g470J/A== + "@types/babel-types@*", "@types/babel-types@^7.0.0": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/babel-types/-/babel-types-7.0.11.tgz#263b113fa396fac4373188d73225297fb86f19a9" @@ -1650,6 +1677,11 @@ dependencies: "@types/webidl-conversions" "*" +"@types/events@^3.0.0": + version "3.0.0" + resolved "https://registry.yarnpkg.com/@types/events/-/events-3.0.0.tgz#2862f3f58a9a7f7c3e78d79f130dd4d71c25c2a7" + integrity sha512-EaObqwIvayI5a8dCzhFrjKzVwKLxjoG9T6Ppd5CEo07LRKfQ8Yokw54r5+Wq7FaBQ+yXRvQAYPrHwya1/UFt9g== + "@types/graceful-fs@^4.1.3": version "4.1.5" resolved "https://registry.yarnpkg.com/@types/graceful-fs/-/graceful-fs-4.1.5.tgz#21ffba0d98da4350db64891f92a9e5db3cdb4e15" @@ -1684,6 +1716,15 @@ expect "^29.0.0" pretty-format "^29.0.0" +"@types/jsdom@^16.2.4": + version "16.2.15" + resolved "https://registry.yarnpkg.com/@types/jsdom/-/jsdom-16.2.15.tgz#6c09990ec43b054e49636cba4d11d54367fc90d6" + integrity sha512-nwF87yjBKuX/roqGYerZZM0Nv1pZDMAT5YhOHYeM/72Fic+VEqJh4nyoqoapzJnW3pUlfxPY5FhgsJtM+dRnQQ== + dependencies: + "@types/node" "*" + "@types/parse5" "^6.0.3" + "@types/tough-cookie" "*" + "@types/json-schema@^7.0.9": version "7.0.11" resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.11.tgz#d421b6c527a3037f7c84433fd2c4229e016863d3" @@ -1727,6 +1768,11 @@ resolved "https://registry.yarnpkg.com/@types/normalize-package-data/-/normalize-package-data-2.4.1.tgz#d3357479a0fdfdd5907fe67e17e0a85c906e1301" integrity sha512-Gj7cI7z+98M282Tqmp2K5EIsoouUEzbBJhQQzDE3jSIRk6r9gsz0oUokqIUR4u1R3dMHo0pDHM7sNOHyhulypw== +"@types/parse5@^6.0.3": + version "6.0.3" + resolved "https://registry.yarnpkg.com/@types/parse5/-/parse5-6.0.3.tgz#705bb349e789efa06f43f128cef51240753424cb" + integrity sha512-SuT16Q1K51EAVPz1K29DJ/sXjhSQ0zjvsypYJ6tlwVsRV9jwW5Adq2ch8Dq8kDBCkYnELS7N7VNCSB5nC56t/g== + "@types/prettier@^2.1.5": version "2.7.1" resolved "https://registry.yarnpkg.com/@types/prettier/-/prettier-2.7.1.tgz#dfd20e2dc35f027cdd6c1908e80a5ddc7499670e" @@ -1737,6 +1783,11 @@ resolved "https://registry.yarnpkg.com/@types/retry/-/retry-0.12.0.tgz#2b35eccfcee7d38cd72ad99232fbd58bffb3c84d" integrity sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA== +"@types/sdp-transform@^2.4.5": + version "2.4.5" + resolved "https://registry.yarnpkg.com/@types/sdp-transform/-/sdp-transform-2.4.5.tgz#3167961e0a1a5265545e278627aa37c606003f53" + integrity sha512-GVO0gnmbyO3Oxm2HdPsYUNcyihZE3GyCY8ysMYHuQGfLhGZq89Nm4lSzULWTzZoyHtg+VO/IdrnxZHPnPSGnAg== + "@types/semver@^7.3.12": version "7.3.13" resolved "https://registry.yarnpkg.com/@types/semver/-/semver-7.3.13.tgz#da4bfd73f49bd541d28920ab0e2bf0ee80f71c91" @@ -1747,6 +1798,11 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c" integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw== +"@types/tough-cookie@*": + version "4.0.2" + resolved "https://registry.yarnpkg.com/@types/tough-cookie/-/tough-cookie-4.0.2.tgz#6286b4c7228d58ab7866d19716f3696e03a09397" + integrity sha512-Q5vtl1W5ue16D+nIaW8JWebSSraJVlK+EthKn7e7UcD4KWsaSJ8BqGPXNaPghgtcn/fhvrN17Tv8ksUsQpiplw== + "@types/webidl-conversions@*": version "7.0.0" resolved "https://registry.yarnpkg.com/@types/webidl-conversions/-/webidl-conversions-7.0.0.tgz#2b8e60e33906459219aa587e9d1a612ae994cfe7" @@ -1860,6 +1916,11 @@ JSONStream@^1.0.3: jsonparse "^1.2.0" through ">=2.2.7 <3" +abab@^2.0.5, abab@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/abab/-/abab-2.0.6.tgz#41b80f2c871d19686216b82309231cfd3cb3d291" + integrity sha512-j2afSsaIENvHZN2B8GOpF566vZ5WVk5opAiMTvWgaQT8DkbOqsTfvNAvHoRGU2zzP8cPoqys+xHTRDWW8L+/BA== + ace-builds@^1.4.13: version "1.12.3" resolved "https://registry.yarnpkg.com/ace-builds/-/ace-builds-1.12.3.tgz#db77530a5854ee38a0c660e9904d2cb29a247c2f" @@ -1872,6 +1933,14 @@ acorn-globals@^3.0.0: dependencies: acorn "^4.0.4" +acorn-globals@^6.0.0: + version "6.0.0" + resolved "https://registry.yarnpkg.com/acorn-globals/-/acorn-globals-6.0.0.tgz#46cdd39f0f8ff08a876619b55f5ac8a6dc770b45" + integrity sha512-ZQl7LOWaF5ePqqcX4hLuv/bLXYQNfNWw2c0/yX/TsPRKamzHcTGQnlCjHT3TsmkOUVEPS3crCxiPfdzE/Trlhg== + dependencies: + acorn "^7.1.1" + acorn-walk "^7.1.1" + acorn-jsx@^5.3.2: version "5.3.2" resolved "https://registry.yarnpkg.com/acorn-jsx/-/acorn-jsx-5.3.2.tgz#7ed5bb55908b3b2f1bc55c6af1653bada7f07937" @@ -1886,7 +1955,7 @@ acorn-node@^1.2.0, acorn-node@^1.3.0, acorn-node@^1.5.2, acorn-node@^1.8.2: acorn-walk "^7.0.0" xtend "^4.0.2" -acorn-walk@^7.0.0: +acorn-walk@^7.0.0, acorn-walk@^7.1.1: version "7.2.0" resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-7.2.0.tgz#0de889a601203909b0fbe07b8938dc21d2e967bc" integrity sha512-OPdCF6GsMIP+Az+aWfAAOEt2/+iVDKE7oy6lJ098aoe59oAmK76qV6Gw60SbZ8jHuG2wH058GF4pLFbYamYrVA== @@ -1901,7 +1970,7 @@ acorn@^4.0.4, acorn@~4.0.2: resolved "https://registry.yarnpkg.com/acorn/-/acorn-4.0.13.tgz#105495ae5361d697bd195c825192e1ad7f253787" integrity sha512-fu2ygVGuMmlzG8ZeRJ0bvR41nsAkxxhbyk8bZ1SS521Z7vmgJFTQQlfz/Mp/nJexGBz+v8sC9bM6+lNgskt4Ug== -acorn@^7.0.0: +acorn@^7.0.0, acorn@^7.1.1: version "7.4.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa" integrity sha512-nQyp0o1/mNdbTO1PO6kHkwSrmgZ0MT/jCCpNiwbUjGoRN4dlBhqJtoQuCnEOKzgTVwg0ZWiCoQy6SxMebQVh8A== @@ -1911,6 +1980,13 @@ acorn@^8.5.0, acorn@^8.8.0: resolved "https://registry.yarnpkg.com/acorn/-/acorn-8.8.0.tgz#88c0187620435c7f6015803f5539dae05a9dbea8" integrity sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w== +agent-base@6: + version "6.0.2" + resolved "https://registry.yarnpkg.com/agent-base/-/agent-base-6.0.2.tgz#49fff58577cfee3f37176feab4c22e00f86d7f77" + integrity sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ== + dependencies: + debug "4" + ajv@^6.10.0, ajv@^6.12.4: version "6.12.6" resolved "https://registry.yarnpkg.com/ajv/-/ajv-6.12.6.tgz#baf5a62e802b07d977034586f8c3baf5adf26df4" @@ -2066,6 +2142,11 @@ ast-types@^0.14.2: dependencies: tslib "^2.0.1" +asynckit@^0.4.0: + version "0.4.0" + resolved "https://registry.yarnpkg.com/asynckit/-/asynckit-0.4.0.tgz#c79ed97f7f34cb8f2ba1bc9790bcc366474b4b79" + integrity sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q== + available-typed-arrays@^1.0.5: version "1.0.5" resolved "https://registry.yarnpkg.com/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz#92f95616501069d07d10edb2fc37d3e1c65123b7" @@ -2286,6 +2367,11 @@ browser-pack@^6.0.1: through2 "^2.0.0" umd "^3.0.0" +browser-process-hrtime@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/browser-process-hrtime/-/browser-process-hrtime-1.0.0.tgz#3c9b4b7d782c8121e56f10106d84c0d0ffc94626" + integrity sha512-9o5UecI3GhkpM6DrXr69PblIuWxPKk9Y0jHBRhdocZ2y7YECBFCsHm79Pr3OyR2AvjhDkabFJaDJMYRazHgsow== + browser-resolve@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/browser-resolve/-/browser-resolve-2.0.0.tgz#99b7304cb392f8d73dba741bb2d7da28c6d7842b" @@ -2703,6 +2789,13 @@ combine-source-map@^0.8.0, combine-source-map@~0.8.0: lodash.memoize "~3.0.3" source-map "~0.5.3" +combined-stream@^1.0.8: + version "1.0.8" + resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" + integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== + dependencies: + delayed-stream "~1.0.0" + commander@^2.19.0, commander@^2.20.0: version "2.20.3" resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33" @@ -2853,6 +2946,23 @@ crypto-browserify@^3.0.0: randombytes "^2.0.0" randomfill "^1.0.3" +cssom@^0.5.0: + version "0.5.0" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.5.0.tgz#d254fa92cd8b6fbd83811b9fbaed34663cc17c36" + integrity sha512-iKuQcq+NdHqlAcwUY0o/HL69XQrUaQdMjmStJ8JFmUaiiQErlhrmuigkg/CU4E2J0IyUKUrMAgl36TvN67MqTw== + +cssom@~0.3.6: + version "0.3.8" + resolved "https://registry.yarnpkg.com/cssom/-/cssom-0.3.8.tgz#9f1276f5b2b463f2114d3f2c75250af8c1a36f4a" + integrity sha512-b0tGHbfegbhPJpxpiBPU2sCkigAqtM9O121le6bbOlgyV+NyGyCmVfJ6QW9eRjz8CpNfWEOYBIMIGRYkLwsIYg== + +cssstyle@^2.3.0: + version "2.3.0" + resolved "https://registry.yarnpkg.com/cssstyle/-/cssstyle-2.3.0.tgz#ff665a0ddbdc31864b09647f34163443d90b0852" + integrity sha512-AZL67abkUzIuvcHqk7c09cezpGNcxUxU4Ioi/05xHk4DQeTkWmGYftIE6ctU6AEt+Gn4n1lDStOtj7FKycP71A== + dependencies: + cssom "~0.3.6" + d@1, d@^1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/d/-/d-1.0.1.tgz#8698095372d58dbee346ffd0c7093f99f8f9eb5a" @@ -2866,11 +2976,27 @@ dash-ast@^1.0.0: resolved "https://registry.yarnpkg.com/dash-ast/-/dash-ast-1.0.0.tgz#12029ba5fb2f8aa6f0a861795b23c1b4b6c27d37" integrity sha512-Vy4dx7gquTeMcQR/hDkYLGUnwVil6vk4FOOct+djUnHOUWt+zJPJAaRIXaAFkPXtJjvlY7o3rfRu0/3hpnwoUA== +data-urls@^3.0.1: + version "3.0.2" + resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-3.0.2.tgz#9cf24a477ae22bcef5cd5f6f0bfbc1d2d3be9143" + integrity sha512-Jy/tj3ldjZJo63sVAvg6LHt2mHvl4V6AgRAmNDtLdm7faqtsx+aJG42rsyCo9JCoRVKwPFzKlIPx3DIibwSIaQ== + dependencies: + abab "^2.0.6" + whatwg-mimetype "^3.0.0" + whatwg-url "^11.0.0" + de-indent@^1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/de-indent/-/de-indent-1.0.2.tgz#b2038e846dc33baa5796128d0804b455b8c1e21d" integrity sha512-e/1zu3xH5MQryN2zdVaF0OrdNLUbvWxzMbi+iNA6Bky7l1RoP8a2fIbRocyHclXt/arDrrR6lL3TqFD9pMQTsg== +debug@4, debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: + version "4.3.4" + resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" + integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== + dependencies: + ms "2.1.2" + debug@^2.6.9: version "2.6.9" resolved "https://registry.yarnpkg.com/debug/-/debug-2.6.9.tgz#5d128515df134ff327e90a4c93f4e077a536341f" @@ -2885,24 +3011,22 @@ debug@^3.2.7: dependencies: ms "^2.1.1" -debug@^4.1.0, debug@^4.1.1, debug@^4.3.2, debug@^4.3.4: - version "4.3.4" - resolved "https://registry.yarnpkg.com/debug/-/debug-4.3.4.tgz#1319f6579357f2338d3337d2cdd4914bb5dcc865" - integrity sha512-PRWFHuSU3eDtQJPvnNY7Jcket1j0t5OuOsFzPPzsekD52Zl8qUfFIPEiswXqIvHWGVHOgX+7G/vCNNhehwxfkQ== - dependencies: - ms "2.1.2" - decamelize@^1.0.0: version "1.2.0" resolved "https://registry.yarnpkg.com/decamelize/-/decamelize-1.2.0.tgz#f6534d15148269b20352e7bee26f501f9a191290" integrity sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA== +decimal.js@^10.3.1: + version "10.3.1" + resolved "https://registry.yarnpkg.com/decimal.js/-/decimal.js-10.3.1.tgz#d8c3a444a9c6774ba60ca6ad7261c3a94fd5e783" + integrity sha512-V0pfhfr8suzyPGOx3nmq4aHqabehUZn6Ch9kyFpV79TGDTWFmHqUqXdabR7QHqxzrYolF4+tVmJhUG4OURg5dQ== + dedent@^0.7.0: version "0.7.0" resolved "https://registry.yarnpkg.com/dedent/-/dedent-0.7.0.tgz#2495ddbaf6eb874abb0e1be9df22d2e5a544326c" integrity sha512-Q6fKUPqnAHAyhiUgFU7BUzLiv0kd8saH9al7tnu5Q/okj6dnupxyTgFIBjVzJATdfIAm9NAsvXNzjaKa+bxVyA== -deep-is@^0.1.3: +deep-is@^0.1.3, deep-is@~0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/deep-is/-/deep-is-0.1.4.tgz#a6f2dce612fadd2ef1f519b73551f17e85199831" integrity sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ== @@ -2930,6 +3054,11 @@ defined@^1.0.0: resolved "https://registry.yarnpkg.com/defined/-/defined-1.0.1.tgz#c0b9db27bfaffd95d6f61399419b893df0f91ebf" integrity sha512-hsBd2qSVCRE+5PmNdHt1uzyrFu5d3RwmFDKzyNZMFq/EwDNJF7Ee5+D5oEKF0hU6LhtoUF1macFvOe4AskQC1Q== +delayed-stream@~1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" + integrity sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ== + deprecation@^2.0.0, deprecation@^2.3.1: version "2.3.1" resolved "https://registry.yarnpkg.com/deprecation/-/deprecation-2.3.1.tgz#6368cbdb40abf3373b525ac87e4a260c3a700919" @@ -3198,6 +3327,18 @@ escape-string-regexp@^4.0.0: resolved "https://registry.yarnpkg.com/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz#14ba83a5d373e3d311e5afca29cf5bfad965bf34" integrity sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA== +escodegen@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/escodegen/-/escodegen-2.0.0.tgz#5e32b12833e8aa8fa35e1bf0befa89380484c7dd" + integrity sha512-mmHKys/C8BFUGI+MAWNcSYoORYLMdPzjrknd2Vc+bUsjN5bXcr8EhrNB+UTqfL1y3I9c4fw2ihgtMPQLBRiQxw== + dependencies: + esprima "^4.0.1" + estraverse "^5.2.0" + esutils "^2.0.2" + optionator "^0.8.1" + optionalDependencies: + source-map "~0.6.1" + eslint-config-google@^0.14.0: version "0.14.0" resolved "https://registry.yarnpkg.com/eslint-config-google/-/eslint-config-google-0.14.0.tgz#4f5f8759ba6e11b424294a219dbfa18c508bcc1a" @@ -3367,7 +3508,7 @@ espree@^9.4.0: acorn-jsx "^5.3.2" eslint-visitor-keys "^3.3.0" -esprima@^4.0.0, esprima@~4.0.0: +esprima@^4.0.0, esprima@^4.0.1, esprima@~4.0.0: version "4.0.1" resolved "https://registry.yarnpkg.com/esprima/-/esprima-4.0.1.tgz#13b04cdb3e6c5d19df91ab6987a8695619b0aa71" integrity sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A== @@ -3418,7 +3559,7 @@ event-emitter@^0.3.5: d "1" es5-ext "~0.10.14" -events@^3.0.0: +events@^3.0.0, events@^3.2.0: version "3.3.0" resolved "https://registry.yarnpkg.com/events/-/events-3.3.0.tgz#31a95ad0a924e2d2c419a813aeb2c4e878ea7400" integrity sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q== @@ -3518,7 +3659,7 @@ fast-json-stable-stringify@^2.0.0, fast-json-stable-stringify@^2.1.0: resolved "https://registry.yarnpkg.com/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz#874bf69c6f404c2b5d99c481341399fd55892633" integrity sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw== -fast-levenshtein@^2.0.6: +fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: version "2.0.6" resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw== @@ -3623,6 +3764,15 @@ foreground-child@^2.0.0: cross-spawn "^7.0.0" signal-exit "^3.0.2" +form-data@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-4.0.0.tgz#93919daeaf361ee529584b9b31664dc12c9fa452" + integrity sha512-ETEklSGi5t0QMZuiXoA/Q6vcnxcLQP5vdugSpuAyi6SVGi2clPPp+xgEhuMaHC+zGgn31Kd235W35f7Hykkaww== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + fs-extra@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9" @@ -3877,6 +4027,13 @@ hosted-git-info@^2.1.4: resolved "https://registry.yarnpkg.com/hosted-git-info/-/hosted-git-info-2.8.9.tgz#dffc0bf9a21c02209090f2aa69429e1414daf3f9" integrity sha512-mxIDAb9Lsm6DoOJ7xH+5+X4y1LU/4Hi50L9C5sIswK3JzULS4bwk1FvjdBgvYR4bzT4tuUQiC15FE2f5HbLvYw== +html-encoding-sniffer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/html-encoding-sniffer/-/html-encoding-sniffer-3.0.0.tgz#2cb1a8cf0db52414776e5b2a7a04d5dd98158de9" + integrity sha512-oWv4T4yJ52iKrufjnyZPkrN0CH3QnrUqdB6In1g5Fe1mia8GmF36gnfNySxoZtxD5+NmYw1EElVXiBk93UeskA== + dependencies: + whatwg-encoding "^2.0.0" + html-escaper@^2.0.0: version "2.0.2" resolved "https://registry.yarnpkg.com/html-escaper/-/html-escaper-2.0.2.tgz#dfd60027da36a36dfcbe236262c00a5822681453" @@ -3887,16 +4044,40 @@ htmlescape@^1.1.0: resolved "https://registry.yarnpkg.com/htmlescape/-/htmlescape-1.1.1.tgz#3a03edc2214bca3b66424a3e7959349509cb0351" integrity sha512-eVcrzgbR4tim7c7soKQKtxa/kQM4TzjnlU83rcZ9bHU6t31ehfV7SktN6McWgwPWg+JYMA/O3qpGxBvFq1z2Jg== +http-proxy-agent@^5.0.0: + version "5.0.0" + resolved "https://registry.yarnpkg.com/http-proxy-agent/-/http-proxy-agent-5.0.0.tgz#5129800203520d434f142bc78ff3c170800f2b43" + integrity sha512-n2hY8YdoRE1i7r6M0w9DIw5GgZN0G25P8zLCRQ8rjXtTU3vsNFBI/vWK/UIeE6g5MUUz6avwAPXmL6Fy9D/90w== + dependencies: + "@tootallnate/once" "2" + agent-base "6" + debug "4" + https-browserify@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/https-browserify/-/https-browserify-1.0.0.tgz#ec06c10e0a34c0f2faf199f7fd7fc78fffd03c73" integrity sha512-J+FkSdyD+0mA0N+81tMotaRMfSL9SGi+xpD3T6YApKsc3bGSXJlfXri3VyFOeYkfLRQisDk1W+jIFFKBeUBbBg== +https-proxy-agent@^5.0.0: + version "5.0.1" + resolved "https://registry.yarnpkg.com/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz#c59ef224a04fe8b754f3db0063a25ea30d0005d6" + integrity sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA== + dependencies: + agent-base "6" + debug "4" + human-signals@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/human-signals/-/human-signals-2.1.0.tgz#dc91fcba42e4d06e4abaed33b3e7a3c02f514ea0" integrity sha512-B4FFZ6q/T2jhhksgkbEW3HBvWIfDW85snkQgawt07S7J5QXTk6BkNV+0yAeZrM5QpMAdYlocGoljn0sJ/WQkFw== +iconv-lite@0.6.3: + version "0.6.3" + resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.6.3.tgz#a52f80bf38da1952eb5c681790719871a1a72501" + integrity sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw== + dependencies: + safer-buffer ">= 2.1.2 < 3.0.0" + ieee754@^1.1.4: version "1.2.1" resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.2.1.tgz#8eb7a10a63fff25d15a57b001586d177d1b0d352" @@ -4132,6 +4313,11 @@ is-plain-object@^5.0.0: resolved "https://registry.yarnpkg.com/is-plain-object/-/is-plain-object-5.0.0.tgz#4427f50ab3429e9025ea7d52e9043a9ef4159344" integrity sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q== +is-potential-custom-element-name@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-potential-custom-element-name/-/is-potential-custom-element-name-1.0.1.tgz#171ed6f19e3ac554394edf78caa05784a45bebb5" + integrity sha512-bCYeRA2rVibKZd+s2625gGnGF/t7DSqDs4dP7CrLA1m7jKWz6pps0LpYLJN8Q64HtmPKJ1hrN3nzPNKFEKOUiQ== + is-promise@^2.0.0, is-promise@^2.2.2: version "2.2.2" resolved "https://registry.yarnpkg.com/is-promise/-/is-promise-2.2.2.tgz#39ab959ccbf9a774cf079f7b40c7a26f763135f1" @@ -4375,6 +4561,20 @@ jest-each@^29.2.1: jest-util "^29.2.1" pretty-format "^29.2.1" +jest-environment-jsdom@^28.1.3: + version "28.1.3" + resolved "https://registry.yarnpkg.com/jest-environment-jsdom/-/jest-environment-jsdom-28.1.3.tgz#2d4e5d61b7f1d94c3bddfbb21f0308ee506c09fb" + integrity sha512-HnlGUmZRdxfCByd3GM2F100DgQOajUBzEitjGqIREcb45kGjZvRrKUdlaF6escXBdcXNl0OBh+1ZrfeZT3GnAg== + dependencies: + "@jest/environment" "^28.1.3" + "@jest/fake-timers" "^28.1.3" + "@jest/types" "^28.1.3" + "@types/jsdom" "^16.2.4" + "@types/node" "*" + jest-mock "^28.1.3" + jest-util "^28.1.3" + jsdom "^19.0.0" + jest-environment-node@^29.2.2: version "29.2.2" resolved "https://registry.yarnpkg.com/jest-environment-node/-/jest-environment-node-29.2.2.tgz#a64b272773870c3a947cd338c25fd34938390bc2" @@ -4479,6 +4679,14 @@ jest-message-util@^29.2.1: slash "^3.0.0" stack-utils "^2.0.3" +jest-mock@^28.1.3: + version "28.1.3" + resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-28.1.3.tgz#d4e9b1fc838bea595c77ab73672ebf513ab249da" + integrity sha512-o3J2jr6dMMWYVH4Lh/NKmDXdosrsJgi4AviS8oXLujcjpCMBb1FMsblDnOXKZKfSiHLxYub1eS0IHuRXsio9eA== + dependencies: + "@jest/types" "^28.1.3" + "@types/node" "*" + jest-mock@^29.0.0, jest-mock@^29.2.2: version "29.2.2" resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-29.2.2.tgz#9045618b3f9d27074bbcf2d55bdca6a5e2e8bca7" @@ -4734,6 +4942,39 @@ jsdoc@^3.6.6: taffydb "2.6.2" underscore "~1.13.2" +jsdom@^19.0.0: + version "19.0.0" + resolved "https://registry.yarnpkg.com/jsdom/-/jsdom-19.0.0.tgz#93e67c149fe26816d38a849ea30ac93677e16b6a" + integrity sha512-RYAyjCbxy/vri/CfnjUWJQQtZ3LKlLnDqj+9XLNnJPgEGeirZs3hllKR20re8LUZ6o1b1X4Jat+Qd26zmP41+A== + dependencies: + abab "^2.0.5" + acorn "^8.5.0" + acorn-globals "^6.0.0" + cssom "^0.5.0" + cssstyle "^2.3.0" + data-urls "^3.0.1" + decimal.js "^10.3.1" + domexception "^4.0.0" + escodegen "^2.0.0" + form-data "^4.0.0" + html-encoding-sniffer "^3.0.0" + http-proxy-agent "^5.0.0" + https-proxy-agent "^5.0.0" + is-potential-custom-element-name "^1.0.1" + nwsapi "^2.2.0" + parse5 "6.0.1" + saxes "^5.0.1" + symbol-tree "^3.2.4" + tough-cookie "^4.0.0" + w3c-hr-time "^1.0.2" + w3c-xmlserializer "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-encoding "^2.0.0" + whatwg-mimetype "^3.0.0" + whatwg-url "^10.0.0" + ws "^8.2.3" + xml-name-validator "^4.0.0" + jsesc@^2.5.1: version "2.5.2" resolved "https://registry.yarnpkg.com/jsesc/-/jsesc-2.5.2.tgz#80564d2e483dacf6e8ef209650a67df3f0c283a4" @@ -4848,6 +5089,14 @@ levn@^0.4.1: prelude-ls "^1.2.1" type-check "~0.4.0" +levn@~0.3.0: + version "0.3.0" + resolved "https://registry.yarnpkg.com/levn/-/levn-0.3.0.tgz#3b09924edf9f083c0490fdd4c0bc4421e04764ee" + integrity sha512-0OO4y2iOHix2W6ujICbKIaEQXvFQHue65vUG3pb5EUomzPI90z9hsA1VsO/dbIIpC53J8gxM9Q4Oho0jrCM/yA== + dependencies: + prelude-ls "~1.1.2" + type-check "~0.3.2" + lines-and-columns@^1.1.6: version "1.2.4" resolved "https://registry.yarnpkg.com/lines-and-columns/-/lines-and-columns-1.2.4.tgz#eca284f75d2965079309dc0ad9255abb2ebc1632" @@ -5006,6 +5255,14 @@ matrix-mock-request@^2.5.0: dependencies: expect "^28.1.0" +matrix-widget-api@^1.0.0: + version "1.1.1" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.1.1.tgz#d3fec45033d0cbc14387a38ba92dac4dbb1be962" + integrity sha512-gNSgmgSwvOsOcWK9k2+tOhEMYBiIMwX95vMZu0JqY7apkM02xrOzUBuPRProzN8CnbIALH7e3GAhatF6QCNvtA== + dependencies: + "@types/events" "^3.0.0" + events "^3.2.0" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" @@ -5060,6 +5317,18 @@ miller-rabin@^4.0.0: bn.js "^4.0.0" brorand "^1.0.1" +mime-db@1.52.0: + version "1.52.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.52.0.tgz#bbabcdc02859f4987301c856e3387ce5ec43bf70" + integrity sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg== + +mime-types@^2.1.12: + version "2.1.35" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.35.tgz#381a871b62a734450660ae3deee44813f70d959a" + integrity sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw== + dependencies: + mime-db "1.52.0" + mimic-fn@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/mimic-fn/-/mimic-fn-2.1.0.tgz#7ed2c2ccccaf84d3ffcb7a69b57711fc2083401b" @@ -5217,6 +5486,11 @@ npm-run-path@^4.0.1: dependencies: path-key "^3.0.0" +nwsapi@^2.2.0: + version "2.2.1" + resolved "https://registry.yarnpkg.com/nwsapi/-/nwsapi-2.2.1.tgz#10a9f268fbf4c461249ebcfe38e359aa36e2577c" + integrity sha512-JYOWTeFoS0Z93587vRJgASD5Ut11fYl5NyihP3KrYBvMe1FRRs6RN7m20SA/16GM4P6hTnZjT+UmDOt38UeXNg== + object-assign@^4.0.1, object-assign@^4.1.0, object-assign@^4.1.1: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" @@ -5282,6 +5556,18 @@ open@^8.4.0: is-docker "^2.1.1" is-wsl "^2.2.0" +optionator@^0.8.1: + version "0.8.3" + resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" + integrity sha512-+IW9pACdk3XWmmTXG8m3upGUJst5XRGzxMRjXzAuJ1XnIFNvfhjjIuYkDvysnPQ7qzqVzLt78BCruntqRhWQbA== + dependencies: + deep-is "~0.1.3" + fast-levenshtein "~2.0.6" + levn "~0.3.0" + prelude-ls "~1.1.2" + type-check "~0.3.2" + word-wrap "~1.2.3" + optionator@^0.9.1: version "0.9.1" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.9.1.tgz#4f236a6373dae0566a6d43e1326674f50c291499" @@ -5399,6 +5685,11 @@ parse-json@^5.0.0, parse-json@^5.2.0: json-parse-even-better-errors "^2.3.0" lines-and-columns "^1.1.6" +parse5@6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/parse5/-/parse5-6.0.1.tgz#e1a1c085c569b3dc08321184f19a39cc27f7c30b" + integrity sha512-Ofn/CTFzRGTTxwpNEs9PP93gXShHcTq255nzRYSKe8AkVpZY7e1fpmTfOyoIvjP5HG7Z2ZM7VS9PPhQGW2pOpw== + patch-package@^6.5.0: version "6.5.0" resolved "https://registry.yarnpkg.com/patch-package/-/patch-package-6.5.0.tgz#feb058db56f0005da59cfa316488321de585e88a" @@ -5524,6 +5815,11 @@ prelude-ls@^1.2.1: resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.2.1.tgz#debc6489d7a6e6b0e7611888cec880337d316396" integrity sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g== +prelude-ls@~1.1.2: + version "1.1.2" + resolved "https://registry.yarnpkg.com/prelude-ls/-/prelude-ls-1.1.2.tgz#21932a549f5e52ffd9a827f570e04be62a97da54" + integrity sha512-ESF23V4SKG6lVSGZgYNpbsiaAkdab6ZgOxe52p7+Kid3W3u3bxR4Vfd/o21dmN7jSt0IwgZ4v5MUd26FEtXE9w== + pretty-format@^28.1.3: version "28.1.3" resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-28.1.3.tgz#c9fba8cedf99ce50963a11b27d982a9ae90970d5" @@ -5587,6 +5883,11 @@ pseudomap@^1.0.2: resolved "https://registry.yarnpkg.com/pseudomap/-/pseudomap-1.0.2.tgz#f052a28da70e618917ef0a8ac34c1ae5a68286b3" integrity sha512-b/YwNhb8lk1Zz2+bXXpS/LK9OisiZZ1SNsSLxN1x2OXVEhW2Ckr/7mWE5vrC1ZTiJlD9g19jWszTmJsB+oEpFQ== +psl@^1.1.33: + version "1.9.0" + resolved "https://registry.yarnpkg.com/psl/-/psl-1.9.0.tgz#d0df2a137f00794565fcaf3b2c00cd09f8d5a5a7" + integrity sha512-E/ZsdU4HLs/68gYzgGTkMicWTLPdAftJLfJFlLUAAKZGkStNU72sZjT66SnMDVOfOWY/YAoiD7Jxa9iHvngcag== + public-encrypt@^4.0.0: version "4.0.3" resolved "https://registry.yarnpkg.com/public-encrypt/-/public-encrypt-4.0.3.tgz#4fcc9d77a07e48ba7527e7cbe0de33d0701331e0" @@ -6064,11 +6365,23 @@ safe-regex@^2.1.1: dependencies: regexp-tree "~0.1.1" -safer-buffer@^2.1.0: +"safer-buffer@>= 2.1.2 < 3.0.0", safer-buffer@^2.1.0: version "2.1.2" resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a" integrity sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg== +saxes@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/saxes/-/saxes-5.0.1.tgz#eebab953fa3b7608dbe94e5dadb15c888fa6696d" + integrity sha512-5LBh1Tls8c9xgGjw3QrMwETmTMVk0oFgvrFSvWx62llR2hcEInrKNZ2GZCCuuy2lvWrdl5jhbpeqc5hRYKFOcw== + dependencies: + xmlchars "^2.2.0" + +sdp-transform@^2.14.1: + version "2.14.1" + resolved "https://registry.yarnpkg.com/sdp-transform/-/sdp-transform-2.14.1.tgz#2bb443583d478dee217df4caa284c46b870d5827" + integrity sha512-RjZyX3nVwJyCuTo5tGPx+PZWkDMCg7oOLpSlhjDdZfwUoNqG1mM8nyj31IGHyaPWXhjbP7cdK3qZ2bmkJ1GzRw== + "semver@2 || 3 || 4 || 5", semver@^5.5.0, semver@^5.6.0: version "5.7.1" resolved "https://registry.yarnpkg.com/semver/-/semver-5.7.1.tgz#a954f931aeba508d307bbf069eff0c01c96116f7" @@ -6402,6 +6715,11 @@ supports-preserve-symlinks-flag@^1.0.0: resolved "https://registry.yarnpkg.com/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz#6eda4bd344a3c94aea376d4cc31bc77311039e09" integrity sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w== +symbol-tree@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/symbol-tree/-/symbol-tree-3.2.4.tgz#430637d248ba77e078883951fb9aa0eed7c63fa2" + integrity sha512-9QNk5KwDF+Bvz+PyObkmSYjI5ksVUYtjW7AU22r2NKcfLJcXp96hkDWU3+XndOsUb+AQ9QhfzfCT2O+CNWT5Tw== + synckit@^0.8.4: version "0.8.4" resolved "https://registry.yarnpkg.com/synckit/-/synckit-0.8.4.tgz#0e6b392b73fafdafcde56692e3352500261d64ec" @@ -6526,6 +6844,15 @@ token-stream@0.0.1: resolved "https://registry.yarnpkg.com/token-stream/-/token-stream-0.0.1.tgz#ceeefc717a76c4316f126d0b9dbaa55d7e7df01a" integrity sha512-nfjOAu/zAWmX9tgwi5NRp7O7zTDUD1miHiB40klUnAh9qnL1iXdgzcz/i5dMaL5jahcBAaSfmNOBBJBLJW8TEg== +tough-cookie@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-4.0.0.tgz#d822234eeca882f991f0f908824ad2622ddbece4" + integrity sha512-tHdtEpQCMrc1YLrMaqXXcj6AxhYi/xgit6mZu1+EDWUn+qhUf8wMQoFIy9NXuq23zAwtcB0t/MjACGR18pcRbg== + dependencies: + psl "^1.1.33" + punycode "^2.1.1" + universalify "^0.1.2" + tr46@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/tr46/-/tr46-2.1.0.tgz#fa87aa81ca5d5941da8cbf1f9b749dc969a4e240" @@ -6533,6 +6860,13 @@ tr46@^2.1.0: dependencies: punycode "^2.1.1" +tr46@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/tr46/-/tr46-3.0.0.tgz#555c4e297a950617e8eeddef633c87d4d9d6cbf9" + integrity sha512-l7FvfAHlcmulp8kr+flpQZmVwtu7nfRV7NZujtN0OqES8EL4O4e0qqzL0DC5gAvx/ZC/9lk6rhcUwYvkBnBnYA== + dependencies: + punycode "^2.1.1" + tr46@~0.0.3: version "0.0.3" resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a" @@ -6609,6 +6943,13 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" +type-check@~0.3.2: + version "0.3.2" + resolved "https://registry.yarnpkg.com/type-check/-/type-check-0.3.2.tgz#5884cab512cf1d355e3fb784f30804b2b520db72" + integrity sha512-ZCmOJdvOWDBYJlzAoFkC+Q0+bUyEOS1ltgp1MGU03fqHG+dbi9tBFU2Rd9QKiDZFAYrhPh2JUf7rZRIuHRKtOg== + dependencies: + prelude-ls "~1.1.2" + type-detect@4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" @@ -6757,7 +7098,7 @@ universal-user-agent@^6.0.0: resolved "https://registry.yarnpkg.com/universal-user-agent/-/universal-user-agent-6.0.0.tgz#3381f8503b251c0d9cd21bc1de939ec9df5480ee" integrity sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w== -universalify@^0.1.0: +universalify@^0.1.0, universalify@^0.1.2: version "0.1.2" resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66" integrity sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg== @@ -6871,6 +7212,20 @@ vue2-ace-editor@^0.0.15: dependencies: brace "^0.11.0" +w3c-hr-time@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/w3c-hr-time/-/w3c-hr-time-1.0.2.tgz#0a89cdf5cc15822df9c360543676963e0cc308cd" + integrity sha512-z8P5DvDNjKDoFIHK7q8r8lackT6l+jo/Ye3HOle7l9nICP9lf1Ci25fy9vHd0JOWewkIFzXIEig3TdKT7JQ5fQ== + dependencies: + browser-process-hrtime "^1.0.0" + +w3c-xmlserializer@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/w3c-xmlserializer/-/w3c-xmlserializer-3.0.0.tgz#06cdc3eefb7e4d0b20a560a5a3aeb0d2d9a65923" + integrity sha512-3WFqGEgSXIyGhOmAFtlicJNMjEps8b1MG31NCA0/vOF9+nKMUW1ckhi9cnNHmf88Rzw5V+dwIwsm2C7X8k9aQg== + dependencies: + xml-name-validator "^4.0.0" + walker@^1.0.8: version "1.0.8" resolved "https://registry.yarnpkg.com/walker/-/walker-1.0.8.tgz#bd498db477afe573dc04185f011d3ab8a8d7653f" @@ -6898,6 +7253,34 @@ webidl-conversions@^7.0.0: resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-7.0.0.tgz#256b4e1882be7debbf01d05f0aa2039778ea080a" integrity sha512-VwddBukDzu71offAQR975unBIGqfKZpM+8ZX6ySk8nYhVoo5CYaZyzt3YBvYtRtO+aoGlqxPg/B87NGVZ/fu6g== +whatwg-encoding@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-2.0.0.tgz#e7635f597fd87020858626805a2729fa7698ac53" + integrity sha512-p41ogyeMUrw3jWclHWTQg1k05DSVXPLcVxRTYsXUk+ZooOCZLcoYgPZ/HL/D/N+uQPOtcp1me1WhBEaX02mhWg== + dependencies: + iconv-lite "0.6.3" + +whatwg-mimetype@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-3.0.0.tgz#5fa1a7623867ff1af6ca3dc72ad6b8a4208beba7" + integrity sha512-nt+N2dzIutVRxARx1nghPKGv1xHikU7HKdfafKkLNLindmPU/ch3U31NOCGGA/dmPcmb1VlofO0vnKAcsm0o/Q== + +whatwg-url@^10.0.0: + version "10.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-10.0.0.tgz#37264f720b575b4a311bd4094ed8c760caaa05da" + integrity sha512-CLxxCmdUby142H5FZzn4D8ikO1cmypvXVQktsgosNy4a4BHrDHeciBBGZhb0bNoR5/MltoCatso+vFjjGx8t0w== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + +whatwg-url@^11.0.0: + version "11.0.0" + resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-11.0.0.tgz#0a849eebb5faf2119b901bb76fd795c2848d4018" + integrity sha512-RKT8HExMpoYx4igMiVMY83lN6UeITKJlBQ+vR/8ZJ8OCdSiN3RwCq+9gH0+Xzj0+5IrM6i4j/6LuvzbZIQgEcQ== + dependencies: + tr46 "^3.0.0" + webidl-conversions "^7.0.0" + whatwg-url@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d" @@ -6965,7 +7348,7 @@ with@^5.0.0: acorn "^3.1.0" acorn-globals "^3.0.0" -word-wrap@^1.2.3: +word-wrap@^1.2.3, word-wrap@~1.2.3: version "1.2.3" resolved "https://registry.yarnpkg.com/word-wrap/-/word-wrap-1.2.3.tgz#610636f6b1f703891bd34771ccb17fb93b47079c" integrity sha512-Hz/mrNwitNRh/HUAtM/VT/5VH+ygD6DV7mYKZAtHOrbs8U7lvPS6xf7EJKMF0uW1KJCl0H701g3ZGus+muE5vQ== @@ -6997,11 +7380,26 @@ write-file-atomic@^4.0.1: imurmurhash "^0.1.4" signal-exit "^3.0.7" +ws@^8.2.3: + version "8.8.1" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.8.1.tgz#5dbad0feb7ade8ecc99b830c1d77c913d4955ff0" + integrity sha512-bGy2JzvzkPowEJV++hF07hAD6niYSr0JzBNo/J29WsB57A2r7Wlc1UFcTR9IzrPvuNVO4B8LGqF8qcpsVOhJCA== + +xml-name-validator@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/xml-name-validator/-/xml-name-validator-4.0.0.tgz#79a006e2e63149a8600f15430f0a4725d1524835" + integrity sha512-ICP2e+jsHvAj2E2lIHxa5tjXRlKDJo4IdvPvCXbXQGdzSfmSpNVyIKMvoZHjDY9DP0zV17iI85o90vRFXNccRw== + xml@1.0.1: version "1.0.1" resolved "https://registry.yarnpkg.com/xml/-/xml-1.0.1.tgz#78ba72020029c5bc87b8a81a3cfcd74b4a2fc1e5" integrity sha512-huCv9IH9Tcf95zuYCsQraZtWnJvBtLVE0QHMOs8bWyZAFZNDcYjsPq1nEx8jKA9y+Beo9v+7OBPRisQTjinQMw== +xmlchars@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/xmlchars/-/xmlchars-2.2.0.tgz#060fe1bcb7f9c76fe2a17db86a9bc3ab894210cb" + integrity sha512-JZnDKK8B0RCDw84FNdDAIpZK+JuJw+s7Lz8nksI7SIuU3UXJJslUthsi+uWBUYOwPFwW7W7PRLRfUKpxjtjFCw== + xmlcreate@^2.0.4: version "2.0.4" resolved "https://registry.yarnpkg.com/xmlcreate/-/xmlcreate-2.0.4.tgz#0c5ab0f99cdd02a81065fa9cd8f8ae87624889be"