Add an audio system to resume AudioContext on mobiles and to enable C…

…hrome AEC (#54)
networked-aframe · Jun 12, 2023 · 712690a · 712690a
1 parent cccebd8
commit 712690a
Show file tree

Hide file tree

Showing 3 changed files with 150 additions and 0 deletions.
diff --git a/examples/audio-with-camera.html b/examples/audio-with-camera.html
@@ -12,6 +12,7 @@
     ></script>
     <script src="https://webrtc.github.io/adapter/adapter-latest.js" crossorigin="anonymous"></script>
     <script src="./dist/naf-janus-adapter.js"></script>
+    <script src="./js/audio-system.js"></script>
 
     <script
       src="https://unpkg.com/aframe-randomizer-components@^3.0.1/dist/aframe-randomizer-components.min.js"
@@ -48,6 +49,7 @@
 
   <body>
     <a-scene
+      audio
       networked-scene="
         room: 1;
         debug: true;

diff --git a/examples/index.html b/examples/index.html
@@ -12,6 +12,7 @@
     ></script>
     <script src="https://webrtc.github.io/adapter/adapter-latest.js" crossorigin="anonymous"></script>
     <script src="./dist/naf-janus-adapter.js"></script>
+    <script src="./js/audio-system.js"></script>
 
     <script
       src="https://unpkg.com/aframe-randomizer-components@^3.0.1/dist/aframe-randomizer-components.min.js"
@@ -48,6 +49,7 @@
 
   <body>
     <a-scene
+      audio
       networked-scene="
         room: 1;
         debug: true;

diff --git a/examples/js/audio-system.js b/examples/js/audio-system.js
@@ -0,0 +1,146 @@
+/* global AFRAME, THREE */
+let delayedReconnectTimeout = null;
+function performDelayedReconnect(gainNode) {
+  if (delayedReconnectTimeout) {
+    clearTimeout(delayedReconnectTimeout);
+  }
+
+  delayedReconnectTimeout = setTimeout(() => {
+    delayedReconnectTimeout = null;
+    console.warn(
+      "enableChromeAEC: recreate RTCPeerConnection loopback because the local connection was disconnected for 10s"
+    );
+    enableChromeAEC(gainNode);
+  }, 10000);
+}
+
+async function enableChromeAEC(gainNode) {
+  /**
+   *  workaround for: https://bugs.chromium.org/p/chromium/issues/detail?id=687574
+   *  1. grab the GainNode from the scene's THREE.AudioListener
+   *  2. disconnect the GainNode from the AudioDestinationNode (basically the audio out), this prevents hearing the audio twice.
+   *  3. create a local webrtc connection between two RTCPeerConnections (see this example: https://webrtc.github.io/samples/src/content/peerconnection/pc1/)
+   *  4. create a new MediaStreamDestination from the scene's THREE.AudioContext and connect the GainNode to it.
+   *  5. add the MediaStreamDestination's track  to one of those RTCPeerConnections
+   *  6. connect the other RTCPeerConnection's stream to a new audio element.
+   *  All audio is now routed through Chrome's audio mixer, thus enabling AEC, while preserving all the audio processing that was performed via the WebAudio API.
+   */
+
+  const audioEl = new Audio();
+  audioEl.setAttribute("autoplay", "autoplay");
+  audioEl.setAttribute("playsinline", "playsinline");
+
+  const context = THREE.AudioContext.getContext();
+  const loopbackDestination = context.createMediaStreamDestination();
+  const outboundPeerConnection = new RTCPeerConnection();
+  const inboundPeerConnection = new RTCPeerConnection();
+
+  const onError = (e) => {
+    console.error("enableChromeAEC: RTCPeerConnection loopback initialization error", e);
+  };
+
+  outboundPeerConnection.addEventListener("icecandidate", (e) => {
+    inboundPeerConnection.addIceCandidate(e.candidate).catch(onError);
+  });
+  outboundPeerConnection.addEventListener("iceconnectionstatechange", (e) => {
+    console.warn(
+      "enableChromeAEC: outboundPeerConnection state changed to " + outboundPeerConnection.iceConnectionState
+    );
+    if (outboundPeerConnection.iceConnectionState === "disconnected") {
+      performDelayedReconnect(gainNode);
+    }
+    if (outboundPeerConnection.iceConnectionState === "connected") {
+      if (delayedReconnectTimeout) {
+        // The RTCPeerConnection reconnected by itself, cancel recreating the
+        // local connection.
+        clearTimeout(delayedReconnectTimeout);
+      }
+    }
+  });
+
+  inboundPeerConnection.addEventListener("icecandidate", (e) => {
+    outboundPeerConnection.addIceCandidate(e.candidate).catch(onError);
+  });
+  inboundPeerConnection.addEventListener("iceconnectionstatechange", (e) => {
+    console.warn("enableChromeAEC: inboundPeerConnection state changed to " + inboundPeerConnection.iceConnectionState);
+    if (inboundPeerConnection.iceConnectionState === "disconnected") {
+      performDelayedReconnect(gainNode);
+    }
+    if (inboundPeerConnection.iceConnectionState === "connected") {
+      if (delayedReconnectTimeout) {
+        // The RTCPeerConnection reconnected by itself, cancel recreating the
+        // local connection.
+        clearTimeout(delayedReconnectTimeout);
+      }
+    }
+  });
+
+  inboundPeerConnection.addEventListener("track", (e) => {
+    audioEl.srcObject = e.streams[0];
+  });
+
+  try {
+    // The following should never fail, but just in case, we won't disconnect/reconnect the gainNode unless all of this succeeds
+    loopbackDestination.stream.getTracks().forEach((track) => {
+      outboundPeerConnection.addTrack(track, loopbackDestination.stream);
+    });
+
+    const offer = await outboundPeerConnection.createOffer();
+    outboundPeerConnection.setLocalDescription(offer);
+    await inboundPeerConnection.setRemoteDescription(offer);
+
+    const answer = await inboundPeerConnection.createAnswer();
+    inboundPeerConnection.setLocalDescription(answer);
+    outboundPeerConnection.setRemoteDescription(answer);
+
+    gainNode.disconnect();
+    gainNode.connect(loopbackDestination);
+  } catch (e) {
+    onError(e);
+  }
+}
+
+AFRAME.registerSystem("audio", {
+  init() {
+    const sceneEl = this.el;
+    this.audioContext = THREE.AudioContext.getContext();
+    this.audioContextNeedsToBeResumed = false;
+
+    /**
+     * Chrome and Safari will start Audio contexts in a "suspended" state.
+     * A user interaction (touch/mouse event) is needed in order to resume the AudioContext.
+     */
+    const resume = () => {
+      this.audioContext.resume();
+
+      setTimeout(() => {
+        if (this.audioContext.state === "running") {
+          if (!AFRAME.utils.device.isMobile() && /chrome/i.test(navigator.userAgent)) {
+            enableChromeAEC(sceneEl.audioListener.gain);
+          }
+
+          document.body.removeEventListener("touchend", resume, false);
+          document.body.removeEventListener("mouseup", resume, false);
+        }
+      }, 0);
+    };
+    this.audioContext.onstatechange = () => {
+      console.log(`AudioContext state changed to ${this.audioContext.state}`);
+      if (this.audioContext.state === "suspended") {
+        // When you unplug the headphone or when the bluetooth headset disconnects on
+        // iOS Safari or Chrome, the state changes to suspended.
+        // Chrome Android doesn't go in suspended state for this case.
+        console.log("The audio has been suspended, click somewhere in the room to resume the audio.");
+        document.body.addEventListener("touchend", resume, false);
+        document.body.addEventListener("mouseup", resume, false);
+        this.audioContextNeedsToBeResumed = true;
+      } else if (this.audioContext.state === "running" && this.audioContextNeedsToBeResumed) {
+        this.audioContextNeedsToBeResumed = false;
+        console.log("The audio is now enabled again.");
+      }
+    };
+
+    document.body.addEventListener("touchend", resume, false);
+    document.body.addEventListener("mouseup", resume, false);
+  },
+});