[Broadcaster] Add support for Simulcast input

broadcaster/assets/js/chat.js

@@ -2,8 +2,6 @@ import { Socket, Presence } from "phoenix"
 
 export async function connectChat() {
   const viewercount = document.getElementById("viewercount");
-  const chatToggler = document.getElementById("chat-toggler");
-  const chat = document.getElementById("chat");
   const chatMessages = document.getElementById("chat-messages");
   const chatInput = document.getElementById("chat-input");
   const chatNickname = document.getElementById("chat-nickname");
@@ -84,22 +82,4 @@ export async function connectChat() {
   chatNickname.onclick = () => {
     chatNickname.classList.remove("invalid-input");
   }
-
-  chatToggler.onclick = () => {
-    if (window.getComputedStyle(chat).display === "none") {
-      chat.style.display = "flex";
-
-      // For screen's width lower than 1024,
-      // eiter show video player or chat at the same time.
-      if (window.innerWidth < 1024) {
-        document.getElementById("videoplayer-wrapper").style.display = "none";
-      }
-    } else {
-      chat.style.display = "none";
-
-      if (window.innerWidth < 1024) {
-        document.getElementById("videoplayer-wrapper").style.display = "block";        
-      }
-    } 
-  }
 }

broadcaster/assets/js/home.js

@@ -1,10 +1,17 @@
 import { connectChat } from "./chat.js"
 
+const chatToggler = document.getElementById("chat-toggler");
+const chat = document.getElementById("chat");
+const settingsToggler = document.getElementById("settings-toggler");
+const settings = document.getElementById("settings");
+const videoQuality = document.getElementById("video-quality");
+
 const pcConfig = { iceServers: [{ urls: "stun:stun.l.google.com:19302" }] };
 const whepEndpoint = `${window.location.origin}/api/whep`
 const videoPlayer = document.getElementById("videoplayer");
 const candidates = [];
 let patchEndpoint;
+let layers = null;
 
 async function sendCandidate(candidate) {
   const response = await fetch(patchEndpoint, {
@@ -58,26 +65,137 @@ async function connectMedia() {
     body: pc.localDescription.sdp
   });
 
-  if (response.status === 201) {
-    patchEndpoint = response.headers.get("location");
-    console.log("Sucessfully initialized WHEP connection")
-
-  } else {
+  if (response.status !== 201) {
     console.error(`Failed to initialize WHEP connection, status: ${response.status}`);
     return;
   }
 
+  patchEndpoint = response.headers.get("location");
+  console.log("Sucessfully initialized WHEP connection")
+
   for (const candidate of candidates) {
     sendCandidate(candidate);
   }
 
   let sdp = await response.text();
   await pc.setRemoteDescription({ type: "answer", sdp: sdp });
+
+  connectServerEvents();
+}
+
+async function connectServerEvents() {
+  const response = await fetch(`${patchEndpoint}/sse`, {
+    method: "POST",
+    cache: "no-cache",
+    headers: { "Content-Type": "application/json" },
+    body: JSON.stringify(["layers"])
+  });
+
+  if (response.status !== 201) {
+    console.error(`Failed to fetch SSE endpoint, status: ${response.status}`);
+    return;
+  }
+
+  const eventStream = response.headers.get("location");
+  const eventSource = new EventSource(eventStream);
+  eventSource.onopen = (ev) => {
+    console.log("EventStream opened", ev);
+  }
+
+  eventSource.onmessage = (ev) => {
+    const data = JSON.parse(ev.data);
+    updateLayers(data.layers)
+  };
+
+  eventSource.onerror = (ev) => {
+    console.log("EventStream closed", ev);
+    eventSource.close();
+  };
+}
+
+function updateLayers(new_layers) {
+  // check if layers changed, if not, just return
+  if (new_layers === null && layers === null) return;
+  if (
+    layers !== null &&
+    new_layers !== null &&
+    new_layers.length === layers.length &&
+    new_layers.every((layer, i) => layer === layers[i])
+  ) return;
+
+  if (new_layers === null) {
+    videoQuality.appendChild(new Option("Disabled", null, true, true));
+    videoQuality.disabled = true;
+    layers = null;
+    return;
+  }
+
+  while (videoQuality.firstChild) {
+    videoQuality.removeChild(videoQuality.firstChild);
+  }
+
+  if (new_layers === null) {
+    videoQuality.appendChild(new Option("Disabled", null, true, true));
+    videoQuality.disabled = true;
+  } else {
+    videoQuality.disabled = false;
+    new_layers
+      .map((layer, i) => {
+        var text = layer;
+        if (layer == "h") text = "High";
+        if (layer == "m") text = "Medium";
+        if (layer == "l") text = "Low";
+        return new Option(text, layer, i == 0, layer == 0);
+      })
+      .forEach(option => videoQuality.appendChild(option))
+  }
+
+  layers = new_layers;
+}
+
+async function changeLayer(layer) {
+  if (patchEndpoint) {
+    const response = await fetch(`${patchEndpoint}/layer`, {
+      method: "POST",
+      cache: "no-cache",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({encodingId: layer})
+    });
+
+    if (response.status != 200) {
+      console.warn("Changing layer failed", response)
+      updateLayers(null);
+    }
+  }
+}
+
+function toggleBox(element, other) {
+    if (window.getComputedStyle(element).display === "none") {
+      element.style.display = "flex";
+      other.style.display = "none";
+
+      // For screen's width lower than 1024,
+      // eiter show video player or chat at the same time.
+      if (window.innerWidth < 1024) {
+        document.getElementById("videoplayer-wrapper").style.display = "none";
+      }
+    } else {
+      element.style.display = "none";
+
+      if (window.innerWidth < 1024) {
+        document.getElementById("videoplayer-wrapper").style.display = "block";
+      }
+    } 
 }
 
 export const Home = {
   mounted() {
     connectMedia()
     connectChat()
+
+    videoQuality.onchange = () => changeLayer(videoQuality.value)
+
+    chatToggler.onclick = () => toggleBox(chat, settings);
+    settingsToggler.onclick = () => toggleBox(settings, chat);
   }
 }

broadcaster/assets/js/player.js

@@ -1,10 +1,14 @@
 const audioDevices = document.getElementById('audioDevices');
 const videoDevices = document.getElementById('videoDevices');
-const maxVideoBitrate = document.getElementById('maxVideoBitrate');
 const serverUrl = document.getElementById('serverUrl');
 const serverToken = document.getElementById('serverToken');
 const button = document.getElementById('button');
 const previewPlayer = document.getElementById('previewPlayer');
+const highVideoBitrate = document.getElementById('highVideoBitrate');
+const mediumVideoBitrate = document.getElementById('mediumVideoBitrate');
+const lowVideoBitrate = document.getElementById('lowVideoBitrate');
+
+const mediaConstraints = {video: {width: {ideal: 1280}, height: {ideal: 720}, frameRate: {ideal: 24}}, audio: true}
 
 let localStream = undefined;
 let pc = undefined;
@@ -61,16 +65,25 @@ async function startStreaming() {
   disableControls();
 
   pc = new RTCPeerConnection();
-  for (const track of localStream.getTracks()) {
-    pc.addTrack(track);
-  }
+  pc.addTrack(localStream.getAudioTracks()[0], localStream);
+  pc.addTransceiver(localStream.getVideoTracks()[0], {
+    streams: [localStream],
+    sendEncodings: [
+      { rid: "h", maxBitrate: 1500 * 1024},
+      { rid: "m", scaleResolutionDownBy: 2, maxBitrate: 600 * 1024},
+      { rid: "l", scaleResolutionDownBy: 4, maxBitrate: 300 * 1024 },
+    ],
+  });
 
   // limit max bitrate
   pc.getSenders()
     .filter((sender) => sender.track.kind === 'video')
     .forEach(async (sender) => {
       const params = sender.getParameters();
-      params.encodings[0].maxBitrate = parseInt(maxVideoBitrate.value) * 1024;
+      console.log(params.encodings);
+      params.encodings.find(e => e.rid === "h").maxBitrate = parseInt(highVideoBitrate.value) * 1024;
+      params.encodings.find(e => e.rid === "m").maxBitrate = parseInt(mediumVideoBitrate.value) * 1024;
+      params.encodings.find(e => e.rid === "l").maxBitrate = parseInt(lowVideoBitrate.value) * 1024;
       await sender.setParameters(params);
     });
 
@@ -115,7 +128,7 @@ function stopStreaming() {
 
 async function run() {
   // ask for permissions
-  localStream = await navigator.mediaDevices.getUserMedia({ video: true, audio: true });
+  localStream = await navigator.mediaDevices.getUserMedia(mediaConstraints);
 
   console.log(`Obtained stream with id: ${localStream.id}`);
 

broadcaster/config/config.exs

@@ -54,7 +54,7 @@ config :phoenix, :json_library, Jason
 
 config :mime, :types, %{
   "application/sdp" => ["sdp"],
-  "application/" => ["trickle-ice-sdpfrag"]
+  "application/trickle-ice-sdpfrag" => ["trickle-ice-sdpfrag"]
 }
 
 config :broadcaster,