Basic call upgrade support

Signed-off-by: Šimon Brandner <[email protected]>

src/webrtc/call.ts

@@ -391,6 +391,19 @@ export class MatrixCall extends EventEmitter {
         });
     }
 
+    public get hasLocalUserMediaAudioTrack(): boolean {
+        return this.localUsermediaStream?.getAudioTracks().length > 0;
+    }
+
+    public get hasRemoteUserMediaAudioTrack(): boolean {
+        return this.getRemoteFeeds().some((feed) => {
+            return (
+                feed.purpose === SDPStreamMetadataPurpose.Usermedia &&
+                feed.stream.getAudioTracks().length > 0
+            );
+        });
+    }
+
     public get localUsermediaFeed(): CallFeed {
         return this.getLocalFeeds().find((feed) => feed.purpose === SDPStreamMetadataPurpose.Usermedia);
     }
@@ -774,6 +787,49 @@ export class MatrixCall extends EventEmitter {
         this.sendVoipEvent(EventType.CallReject, {});
     }
 
+    /**
+     * Adds a video track to a voice call
+     * @returns new mute state
+     */
+    private async upgradeCall(
+        audio: boolean, video: boolean,
+    ): Promise<void> {
+        // We don't do call downgrades
+        if (!audio && !video) return;
+        if (!this.opponentSupportsSDPStreamMetadata()) return;
+
+        try {
+            const upgradeAudio = audio && !this.hasLocalUserMediaAudioTrack;
+            const upgradeVideo = video && !this.hasLocalUserMediaVideoTrack;
+            logger.debug(`Upgrading call audio=${upgradeAudio} video=${upgradeVideo}`);
+
+            const stream = await this.client.getMediaHandler().getUserMediaStream(upgradeAudio, upgradeVideo);
+            if (upgradeAudio && upgradeVideo) {
+                if (this.hasLocalUserMediaAudioTrack) return;
+                if (this.hasLocalUserMediaVideoTrack) return;
+
+                this.pushLocalFeed(stream, SDPStreamMetadataPurpose.Usermedia);
+            } else if (upgradeAudio) {
+                if (this.hasLocalUserMediaAudioTrack) return;
+
+                const audioTrack = stream.getAudioTracks()[0];
+                this.localUsermediaStream.addTrack(audioTrack);
+                this.peerConn.addTrack(audioTrack, this.localUsermediaStream);
+            } else if (upgradeVideo) {
+                if (this.hasLocalUserMediaVideoTrack) return;
+
+                const videoTrack = stream.getVideoTracks()[0];
+                this.localUsermediaStream.addTrack(videoTrack);
+                this.peerConn.addTrack(videoTrack, this.localUsermediaStream);
+            }
+        } catch (error) {
+            logger.error("Failed to upgrade the call", error);
+            this.emit(CallEvent.Error,
+                new CallError(CallErrorCode.NoUserMedia, "Failed to get camera access: ", error),
+            );
+        }
+    }
+
     /**
      * Returns true if this.remoteSDPStreamMetadata is defined, otherwise returns false
      * @returns {boolean} can screenshare
@@ -888,10 +944,16 @@ export class MatrixCall extends EventEmitter {
     /**
      * Set whether our outbound video should be muted or not.
      * @param {boolean} muted True to mute the outbound video.
+     * @returns the new mute state
      */
-    public setLocalVideoMuted(muted: boolean): void {
+    public async setLocalVideoMuted(muted: boolean): Promise<boolean> {
+        if (!this.hasLocalUserMediaVideoTrack && !muted) {
+            await this.upgradeCall(false, true);
+            return this.isLocalVideoMuted();
+        }
         this.localUsermediaFeed?.setVideoMuted(muted);
         this.updateMuteStatus();
+        return this.isLocalVideoMuted();
     }
 
     /**
@@ -910,8 +972,13 @@ export class MatrixCall extends EventEmitter {
     /**
      * Set whether the microphone should be muted or not.
      * @param {boolean} muted True to mute the mic.
+     * @returns the new mute state
      */
-    public setMicrophoneMuted(muted: boolean): void {
+    public async setMicrophoneMuted(muted: boolean): Promise<boolean> {
+        if (!this.hasLocalUserMediaAudioTrack && !muted) {
+            await this.upgradeCall(false, true);
+            return this.isMicrophoneMuted();
+        }
         this.localUsermediaFeed?.setAudioMuted(muted);
         this.updateMuteStatus();
     }

src/webrtc/callFeed.ts

@@ -48,6 +48,7 @@ export class CallFeed extends EventEmitter {
         private videoMuted: boolean,
     ) {
         super();
+        this.updateStream(null, stream);
 
         if (this.hasAudioTrack) {
             this.initVolumeMeasuring();
@@ -58,6 +59,25 @@ export class CallFeed extends EventEmitter {
         return this.stream.getAudioTracks().length > 0;
     }
 
+    private updateStream(oldStream: MediaStream, newStream: MediaStream): void {
+        if (oldStream) {
+            oldStream.removeEventListener("addtrack", this.onAddTrack);
+            this.measureVolumeActivity(false);
+        }
+        if (newStream) {
+            this.stream = newStream;
+            newStream.addEventListener("addtrack", this.onAddTrack);
+
+            if (this.hasAudioTrack) {
+                this.initVolumeMeasuring();
+            } else {
+                this.measureVolumeActivity(false);
+            }
+        }
+
+        this.emit(CallFeedEvent.NewStream, this.stream);
+    }
+
     private initVolumeMeasuring(): void {
         const AudioContext = window.AudioContext || window.webkitAudioContext;
         if (!this.hasAudioTrack || !AudioContext) return;
@@ -74,6 +94,10 @@ export class CallFeed extends EventEmitter {
         this.frequencyBinCount = new Float32Array(this.analyser.frequencyBinCount);
     }
 
+    private onAddTrack = (): void => {
+        this.emit(CallFeedEvent.NewStream, this.stream);
+    };
+
     /**
      * Returns callRoom member
      * @returns member of the callRoom
@@ -116,14 +140,7 @@ export class CallFeed extends EventEmitter {
      * @param newStream new stream with which to replace the current one
      */
     public setNewStream(newStream: MediaStream): void {
-        this.stream = newStream;
-        this.emit(CallFeedEvent.NewStream, this.stream);
-
-        if (this.hasAudioTrack) {
-            this.initVolumeMeasuring();
-        } else {
-            this.measureVolumeActivity(false);
-        }
+        this.updateStream(this.stream, newStream);
     }
 
     /**