[Recognizer] Use jitter buffer

recognizer/lib/recognizer/room.ex

@@ -6,10 +6,11 @@ defmodule Recognizer.Room do
   require Logger
 
   alias ExWebRTC.{ICECandidate, PeerConnection, RTPCodecParameters, SessionDescription}
-  alias ExWebRTC.RTP.Depayloader
+  alias ExWebRTC.RTP.{Depayloader, JitterBuffer}
 
   @max_session_time_s Application.compile_env!(:recognizer, :max_session_time_s)
   @session_time_timer_interval_ms 1_000
+  @jitter_buffer_latency_ms 50
 
   @video_codecs [
     %RTPCodecParameters{
@@ -43,6 +44,7 @@ defmodule Recognizer.Room do
     Process.send_after(self(), :session_time, @session_time_timer_interval_ms)
 
     {:ok, video_depayloader} = @video_codecs |> hd() |> Depayloader.new()
+    video_buffer = JitterBuffer.new(latency: @jitter_buffer_latency_ms)
 
     {:ok,
      %{
@@ -53,6 +55,7 @@ defmodule Recognizer.Room do
        video_track: nil,
        video_depayloader: video_depayloader,
        video_decoder: Xav.Decoder.new(:vp8),
+       video_buffer: video_buffer,
        audio_track: nil,
        session_start_time: System.monotonic_time(:millisecond)
      }}
@@ -133,24 +136,9 @@ defmodule Recognizer.Room do
         {:ex_webrtc, _pc, {:rtp, track_id, nil, packet}},
         %{video_track: %{id: track_id}} = state
       ) do
-    {frame, depayloader} = Depayloader.depayload(state.video_depayloader, packet)
-    state = %{state | video_depayloader: depayloader}
-
-    with true <- is_nil(state.task),
-         false <- is_nil(frame),
-         {:ok, frame} <- Xav.Decoder.decode(state.video_decoder, frame) do
-      tensor = Xav.Frame.to_nx(frame)
-      task = Task.async(fn -> Nx.Serving.batched_run(Recognizer.VideoServing, tensor) end)
-      state = %{state | task: task}
-      {:noreply, state}
-    else
-      other when other in [:ok, true, false] ->
-        {:noreply, state}
-
-      {:error, :no_keyframe} ->
-        Logger.warning("Couldn't decode video frame - missing keyframe!")
-        {:noreply, state}
-    end
+    state.video_buffer
+    |> JitterBuffer.place_packet(packet)
+    |> handle_jitter_buffer_result(state)
   end
 
   @impl true
@@ -162,6 +150,13 @@ defmodule Recognizer.Room do
     {:noreply, state}
   end
 
+  @impl true
+  def handle_info(:jitter_buffer_timer, state) do
+    state.video_buffer
+    |> JitterBuffer.handle_timer()
+    |> handle_jitter_buffer_result(state)
+  end
+
   @impl true
   def handle_info({:DOWN, _ref, :process, pid, _reason}, state) do
     if pid != state.channel do
@@ -207,4 +202,32 @@ defmodule Recognizer.Room do
   def handle_info(_msg, state) do
     {:noreply, state}
   end
+
+  defp handle_jitter_buffer_result({buffer, packets, timer}, state) do
+    unless is_nil(timer), do: Process.send_after(self(), :jitter_buffer_timer, timer)
+
+    state = Enum.reduce(packets, state, fn packet, state -> handle_packet(packet, state) end)
+
+    {:noreply, %{state | video_buffer: buffer}}
+  end
+
+  defp handle_packet(packet, state) do
+    {frame, depayloader} = Depayloader.depayload(state.video_depayloader, packet)
+    state = %{state | video_depayloader: depayloader}
+
+    with true <- is_nil(state.task),
+         false <- is_nil(frame),
+         {:ok, frame} <- Xav.Decoder.decode(state.video_decoder, frame) do
+      tensor = Xav.Frame.to_nx(frame)
+      task = Task.async(fn -> Nx.Serving.batched_run(Recognizer.VideoServing, tensor) end)
+      %{state | task: task}
+    else
+      other when other in [:ok, true, false] ->
+        state
+
+      {:error, :no_keyframe} ->
+        Logger.warning("Couldn't decode video frame - missing keyframe!")
+        state
+    end
+  end
 end

recognizer/lib/recognizer_web.ex

@@ -43,7 +43,7 @@ defmodule RecognizerWeb do
         layouts: [html: RecognizerWeb.Layouts]
 
       import Plug.Conn
-      import RecognizerWeb.Gettext
+      use Gettext, backend: RecognizerWeb.Gettext
 
       unquote(verified_routes())
     end
@@ -85,7 +85,7 @@ defmodule RecognizerWeb do
       import Phoenix.HTML
       # Core UI components and translation
       import RecognizerWeb.CoreComponents
-      import RecognizerWeb.Gettext
+      use Gettext, backend: RecognizerWeb.Gettext
 
       # Shortcut for generating JS commands
       alias Phoenix.LiveView.JS

recognizer/lib/recognizer_web/components/core_components.ex

@@ -17,7 +17,7 @@ defmodule RecognizerWeb.CoreComponents do
   use Phoenix.Component
 
   alias Phoenix.LiveView.JS
-  import RecognizerWeb.Gettext
+  use Gettext, backend: RecognizerWeb.Gettext
 
   @doc """
   Renders a modal.

recognizer/lib/recognizer_web/gettext.ex

@@ -20,5 +20,5 @@ defmodule RecognizerWeb.Gettext do
 
   See the [Gettext Docs](https://hexdocs.pm/gettext) for detailed usage.
   """
-  use Gettext, otp_app: :recognizer
+  use Gettext.Backend, otp_app: :recognizer
 end

recognizer/mix.exs

@@ -59,7 +59,9 @@ defmodule Recognizer.MixProject do
       {:jason, "~> 1.2"},
       {:dns_cluster, "~> 0.1.1"},
       {:plug_cowboy, "~> 2.5"},
-      {:ex_webrtc, "~> 0.4.0"},
+      # {:ex_webrtc, "~> 0.4.0"},
+      {:ex_webrtc,
+       github: "elixir-webrtc/ex_webrtc", branch: "sgfn/jitter-buffer", override: true},
       {:ex_webrtc_dashboard, "~> 0.4.0"},
       {:xav, "~> 0.5.0"},
       {:bumblebee, "~> 0.5.3"},

recognizer/mix.lock

@@ -20,29 +20,29 @@
   "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"},
   "esbuild": {:hex, :esbuild, "0.8.1", "0cbf919f0eccb136d2eeef0df49c4acf55336de864e63594adcea3814f3edf41", [:mix], [{:castore, ">= 0.0.0", [hex: :castore, repo: "hexpm", optional: false]}, {:jason, "~> 1.4", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "25fc876a67c13cb0a776e7b5d7974851556baeda2085296c14ab48555ea7560f"},
   "ex_dtls": {:hex, :ex_dtls, "0.15.2", "6c8c0f8eb67525216551bd3e0322ab33c9d851d56ef3e065efab4fd277a8fbb9", [:mix], [{:unifex, "~> 1.0", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "6b852bc926bbdc9c1b9c4ecc6cfc73a89d4e106042802cefea2c1503072a9f2a"},
-  "ex_ice": {:hex, :ex_ice, "0.8.0", "f9bd181e8fd2f8ac9a808587ee8a47bf667143069d75f6e4892a62156d798aa7", [:mix], [{:elixir_uuid, "~> 1.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:ex_stun, "~> 0.2.0", [hex: :ex_stun, repo: "hexpm", optional: false]}, {:ex_turn, "~> 0.1.0", [hex: :ex_turn, repo: "hexpm", optional: false]}], "hexpm", "b0476f6b18986f6df48fda4cecb3be5022323572790d1bb49da10b177c936b4e"},
+  "ex_ice": {:hex, :ex_ice, "0.8.1", "4d5c911766ce92e13323b632a55d9ab821092f13fc2ebf236dc233c8c1f9a64c", [:mix], [{:elixir_uuid, "~> 1.0", [hex: :elixir_uuid, repo: "hexpm", optional: false]}, {:ex_stun, "~> 0.2.0", [hex: :ex_stun, repo: "hexpm", optional: false]}, {:ex_turn, "~> 0.1.0", [hex: :ex_turn, repo: "hexpm", optional: false]}], "hexpm", "8f10134e2eb7e6aebbf8fba0d5fcec56d8f8db3e94c3dde045feb463979c2dda"},
   "ex_libsrtp": {:hex, :ex_libsrtp, "0.7.2", "211bd89c08026943ce71f3e2c0231795b99cee748808ed3ae7b97cd8d2450b6b", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:bundlex, "~> 1.3", [hex: :bundlex, repo: "hexpm", optional: false]}, {:membrane_precompiled_dependency_provider, "~> 0.1.0", [hex: :membrane_precompiled_dependency_provider, repo: "hexpm", optional: false]}, {:unifex, "~> 1.1", [hex: :unifex, repo: "hexpm", optional: false]}], "hexpm", "2e20645d0d739a4ecdcf8d4810a0c198120c8a2f617f2b75b2e2e704d59f492a"},
   "ex_rtcp": {:hex, :ex_rtcp, "0.4.0", "f9e515462a9581798ff6413583a25174cfd2101c94a2ebee871cca7639886f0a", [:mix], [], "hexpm", "28956602cf210d692fcdaf3f60ca49681634e1deb28ace41246aee61ee22dc3b"},
   "ex_rtp": {:hex, :ex_rtp, "0.4.0", "1f1b5c1440a904706011e3afbb41741f5da309ce251cb986690ce9fd82636658", [:mix], [], "hexpm", "0f72d80d5953a62057270040f0f1ee6f955c08eeae82ac659c038001d7d5a790"},
   "ex_sdp": {:hex, :ex_sdp, "1.0.0", "c66cd66d60ad03ff1eecdc6db6a1b8a7b89fec260fcc22e8d6703fc5bbf430a3", [:mix], [{:bunch, "~> 1.3", [hex: :bunch, repo: "hexpm", optional: false]}, {:elixir_uuid, "~> 1.2", [hex: :elixir_uuid, repo: "hexpm", optional: false]}], "hexpm", "e165dff988b8ab9d93588636aa5f3f683e1f848fc63b78b12382c8fa3dd39216"},
   "ex_stun": {:hex, :ex_stun, "0.2.0", "feb1fc7db0356406655b2a617805e6c712b93308c8ea2bf0ba1197b1f0866deb", [:mix], [], "hexpm", "1e01ba8290082ccbf37acaa5190d1f69b51edd6de2026a8d6d51368b29d115d0"},
   "ex_turn": {:hex, :ex_turn, "0.1.0", "177405aadf3d754567d0d37cf881a83f9cacf8f45314d188633b04c4a9e7c1ec", [:mix], [{:ex_stun, "~> 0.2.0", [hex: :ex_stun, repo: "hexpm", optional: false]}], "hexpm", "d677737fb7d45274d5dac19fe3c26b9038b6effbc0a6b3e7417bccc76b6d1cd3"},
-  "ex_webrtc": {:hex, :ex_webrtc, "0.4.0", "53fe12c7d23a3f9a8f0906c363b31c4095ebd087d54044eca4bccf3c5bd55424", [:mix], [{:crc, "~> 0.10", [hex: :crc, repo: "hexpm", optional: false]}, {:ex_dtls, "~> 0.15.0", [hex: :ex_dtls, repo: "hexpm", optional: false]}, {:ex_ice, "~> 0.8.0", [hex: :ex_ice, repo: "hexpm", optional: false]}, {:ex_libsrtp, "~> 0.7.1", [hex: :ex_libsrtp, repo: "hexpm", optional: false]}, {:ex_rtcp, "~> 0.4.0", [hex: :ex_rtcp, repo: "hexpm", optional: false]}, {:ex_rtp, "~> 0.4.0", [hex: :ex_rtp, repo: "hexpm", optional: false]}, {:ex_sdp, "~> 1.0", [hex: :ex_sdp, repo: "hexpm", optional: false]}], "hexpm", "f36d84845d68ad5f64e7a675ce32cde88b0f768c1572d3232eb86f4019c7a841"},
+  "ex_webrtc": {:git, "https://github.com/elixir-webrtc/ex_webrtc.git", "8173dc1c82b6515fddf6776ab6c3ae57dbfe8add", [branch: "sgfn/jitter-buffer"]},
   "ex_webrtc_dashboard": {:hex, :ex_webrtc_dashboard, "0.4.0", "c232048c4424e06dcbcb77437ed726895edf6592039b510377e6a6d4fc088f01", [:mix], [{:ex_webrtc, "~> 0.4.0", [hex: :ex_webrtc, repo: "hexpm", optional: false]}, {:phoenix_live_dashboard, "~> 0.8.3", [hex: :phoenix_live_dashboard, repo: "hexpm", optional: false]}], "hexpm", "b3c326f836614f373516fbc77c6951b3c5e481b2cf0cfd7d681748dbaaf918df"},
   "exla": {:hex, :exla, "0.7.3", "51310270a0976974fc758f7b28ebd6ca8e099b3d6fc78b0d484c808e977cb914", [:make, :mix], [{:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:nx, "~> 0.7.1", [hex: :nx, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:xla, "~> 0.6.0", [hex: :xla, repo: "hexpm", optional: false]}], "hexpm", "5b3d5741a24aada21d3b0feb4b99d1fc3c8457f995a63ea16684d8d5678b96ff"},
   "expo": {:hex, :expo, "1.0.0", "647639267e088717232f4d4451526e7a9de31a3402af7fcbda09b27e9a10395a", [:mix], [], "hexpm", "18d2093d344d97678e8a331ca0391e85d29816f9664a25653fd7e6166827827c"},
-  "file_system": {:hex, :file_system, "1.0.0", "b689cc7dcee665f774de94b5a832e578bd7963c8e637ef940cd44327db7de2cd", [:mix], [], "hexpm", "6752092d66aec5a10e662aefeed8ddb9531d79db0bc145bb8c40325ca1d8536d"},
+  "file_system": {:hex, :file_system, "1.0.1", "79e8ceaddb0416f8b8cd02a0127bdbababe7bf4a23d2a395b983c1f8b3f73edd", [:mix], [], "hexpm", "4414d1f38863ddf9120720cd976fce5bdde8e91d8283353f0e31850fa89feb9e"},
   "finch": {:hex, :finch, "0.18.0", "944ac7d34d0bd2ac8998f79f7a811b21d87d911e77a786bc5810adb75632ada4", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: false]}, {:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.3", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 0.2.6 or ~> 1.0", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "69f5045b042e531e53edc2574f15e25e735b522c37e2ddb766e15b979e03aa65"},
   "floki": {:hex, :floki, "0.36.2", "a7da0193538c93f937714a6704369711998a51a6164a222d710ebd54020aa7a3", [:mix], [], "hexpm", "a8766c0bc92f074e5cb36c4f9961982eda84c5d2b8e979ca67f5c268ec8ed580"},
-  "gettext": {:hex, :gettext, "0.25.0", "98a95a862a94e2d55d24520dd79256a15c87ea75b49673a2e2f206e6ebc42e5d", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "38e5d754e66af37980a94fb93bb20dcde1d2361f664b0a19f01e87296634051f"},
+  "gettext": {:hex, :gettext, "0.26.1", "38e14ea5dcf962d1fc9f361b63ea07c0ce715a8ef1f9e82d3dfb8e67e0416715", [:mix], [{:expo, "~> 0.5.1 or ~> 1.0", [hex: :expo, repo: "hexpm", optional: false]}], "hexpm", "01ce56f188b9dc28780a52783d6529ad2bc7124f9744e571e1ee4ea88bf08734"},
+  "heap": {:hex, :heap, "3.0.0", "c6dbcd6e9a0b021432176e89cfd829dd065bd6c115981fdcd981a4251fff5fde", [:mix], [], "hexpm", "373eaca5787e2a2b009c42338e70414f590ceabcf96cfc786627ed762ad4dfc6"},
   "heroicons": {:git, "https://github.com/tailwindlabs/heroicons.git", "88ab3a0d790e6a47404cba02800a6b25d2afae50", [tag: "v2.1.1", sparse: "optimized"]},
   "hpax": {:hex, :hpax, "1.0.0", "28dcf54509fe2152a3d040e4e3df5b265dcb6cb532029ecbacf4ce52caea3fd2", [:mix], [], "hexpm", "7f1314731d711e2ca5fdc7fd361296593fc2542570b3105595bb0bc6d0fad601"},
   "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"},
   "membrane_precompiled_dependency_provider": {:hex, :membrane_precompiled_dependency_provider, "0.1.2", "8af73b7dc15ba55c9f5fbfc0453d4a8edfb007ade54b56c37d626be0d1189aba", [:mix], [{:bundlex, "~> 1.4", [hex: :bundlex, repo: "hexpm", optional: false]}], "hexpm", "7fe3e07361510445a29bee95336adde667c4162b76b7f4c8af3aeb3415292023"},
   "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"},
   "mint": {:hex, :mint, "1.6.2", "af6d97a4051eee4f05b5500671d47c3a67dac7386045d87a904126fd4bbcea2e", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "5ee441dffc1892f1ae59127f74afe8fd82fda6587794278d924e4d90ea3d63f9"},
   "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"},
-  "nimble_ownership": {:hex, :nimble_ownership, "0.3.1", "99d5244672fafdfac89bfad3d3ab8f0d367603ce1dc4855f86a1c75008bce56f", [:mix], [], "hexpm", "4bf510adedff0449a1d6e200e43e57a814794c8b5b6439071274d248d272a549"},
   "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"},
   "nx": {:hex, :nx, "0.7.3", "51ff45d9f9ff58b616f4221fa54ccddda98f30319bb8caaf86695234a469017a", [:mix], [{:complex, "~> 0.5", [hex: :complex, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "5ff29af84f08db9bda66b8ef7ce92ab583ab4f983629fe00b479f1e5c7c705a6"},
   "nx_image": {:hex, :nx_image, "0.1.2", "0c6e3453c1dc30fc80c723a54861204304cebc8a89ed3b806b972c73ee5d119d", [:mix], [{:nx, "~> 0.4", [hex: :nx, repo: "hexpm", optional: false]}], "hexpm", "9161863c42405ddccb6dbbbeae078ad23e30201509cc804b3b3a7c9e98764b81"},