Use H264 codec for screen sharing

zed-industries · Jun 26, 2024 · 0ac44ab · 0ac44ab
1 parent 8d739e0
commit 0ac44ab
Show file tree

Hide file tree

Showing 8 changed files with 44 additions and 18 deletions.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -317,8 +317,7 @@ itertools = "0.11.0"
 lazy_static = "1.4.0"
 libc = "0.2"
 linkify = "0.10.0"
-# livekit = { git = "https://github.com/livekit/rust-sdks", rev = "9c2a467751f1698532393bee95b96cba71af78b3", features = ["dispatcher", "native-tls"], default-features = false }
-livekit = { path = "../livekit-rust-sdks/livekit", features = ["dispatcher", "native-tls"], default-features = false }
+livekit = { git = "https://github.com/livekit/rust-sdks", rev = "b0391a8a9644344731d7ae43289418c9134e2de0", features = ["dispatcher", "native-tls"], default-features = false }
 log = { version = "0.4.16", features = ["kv_unstable_serde"] }
 markup5ever_rcdom = "0.3.0"
 nanoid = "0.4"

diff --git a/crates/call/src/participant.rs b/crates/call/src/participant.rs
@@ -1,14 +1,13 @@
 use anyhow::{anyhow, Result};
-use client::ParticipantIndex;
-use client::{proto, User};
+use client::{proto, ParticipantIndex, User};
 use collections::HashMap;
 use gpui::WeakModel;
 use project::Project;
 use std::sync::Arc;
 
 pub use live_kit_client::{
+    id::TrackSid,
     track::{RemoteAudioTrack, RemoteVideoTrack},
-    id::TrackSid
 };
 
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]

diff --git a/crates/call/src/room.rs b/crates/call/src/room.rs
@@ -17,13 +17,17 @@ use gpui::{
 use language::LanguageRegistry;
 use live_kit_client as livekit;
 use livekit::{
-    capture_local_audio_track, capture_local_video_track, id::ParticipantIdentity,
-    options::TrackPublishOptions, publication::LocalTrackPublication, RoomEvent, RoomOptions,
+    capture_local_audio_track, capture_local_video_track,
+    id::ParticipantIdentity,
+    options::{TrackPublishOptions, VideoCodec},
+    publication::LocalTrackPublication,
+    track::TrackSource,
+    RoomEvent, RoomOptions,
 };
 use postage::{sink::Sink, stream::Stream, watch};
 use project::Project;
 use settings::Settings as _;
-use std::{future::Future, mem, sync::Arc, time::Duration};
+use std::{any::Any, future::Future, mem, sync::Arc, time::Duration};
 use util::{post_inc, ResultExt, TryFutureExt};
 
 pub const RECONNECT_TIMEOUT: Duration = Duration::from_secs(30);
@@ -1045,12 +1049,18 @@ impl Room {
             RoomEvent::LocalTrackUnpublished { publication, .. } => {
                 log::info!("unpublished track {}", publication.sid());
                 if let Some(room) = &mut self.live_kit {
-                    if let LocalTrack::Published { track_publication } = &room.microphone_track {
+                    if let LocalTrack::Published {
+                        track_publication, ..
+                    } = &room.microphone_track
+                    {
                         if track_publication.sid() == publication.sid() {
                             room.microphone_track = LocalTrack::None;
                         }
                     }
-                    if let LocalTrack::Published { track_publication } = &room.screen_track {
+                    if let LocalTrack::Published {
+                        track_publication, ..
+                    } = &room.screen_track
+                    {
                         if track_publication.sid() == publication.sid() {
                             room.screen_track = LocalTrack::None;
                         }
@@ -1328,7 +1338,10 @@ impl Room {
             let publication = participant
                 .publish_track(
                     livekit::track::LocalTrack::Audio(track),
-                    TrackPublishOptions::default(),
+                    TrackPublishOptions {
+                        source: TrackSource::Microphone,
+                        ..Default::default()
+                    },
                 )
                 .await
                 .map_err(|error| anyhow!("failed to publish track: {error}"));
@@ -1361,6 +1374,7 @@ impl Room {
                             }
                             live_kit.microphone_track = LocalTrack::Published {
                                 track_publication: publication,
+                                _stream: Box::new(stream),
                             };
                             cx.notify();
                         }
@@ -1403,12 +1417,16 @@ impl Room {
             let sources = sources.await??;
             let source = sources.first().ok_or_else(|| anyhow!("no display found"))?;
 
-            let (track, _stream) = capture_local_video_track(&**source).await?;
+            let (track, stream) = capture_local_video_track(&**source).await?;
 
             let publication = participant
                 .publish_track(
                     livekit::track::LocalTrack::Video(track),
-                    TrackPublishOptions::default(),
+                    TrackPublishOptions {
+                        source: TrackSource::Screenshare,
+                        video_codec: VideoCodec::H264,
+                        ..Default::default()
+                    },
                 )
                 .await
                 .map_err(|error| anyhow!("error publishing screen track {error:?}"));
@@ -1439,6 +1457,7 @@ impl Room {
                         } else {
                             live_kit.screen_track = LocalTrack::Published {
                                 track_publication: publication,
+                                _stream: Box::new(stream),
                             };
                             cx.notify();
                         }
@@ -1573,7 +1592,9 @@ impl Room {
                 }
             }
             LocalTrack::Pending { .. } => None,
-            LocalTrack::Published { track_publication } => {
+            LocalTrack::Published {
+                track_publication, ..
+            } => {
                 if should_mute {
                     track_publication.mute()
                 } else {
@@ -1634,6 +1655,7 @@ enum LocalTrack {
     },
     Published {
         track_publication: LocalTrackPublication,
+        _stream: Box<dyn Any>,
     },
 }
 

diff --git a/crates/gpui/src/platform/mac.rs b/crates/gpui/src/platform/mac.rs
@@ -39,7 +39,6 @@ pub(crate) use dispatcher::*;
 pub(crate) use display::*;
 pub(crate) use display_link::*;
 pub(crate) use platform::*;
-pub(crate) use screen_capture::*;
 pub(crate) use text_system::*;
 pub(crate) use window::*;
 

diff --git a/crates/live_kit_client/examples/test_app.rs b/crates/live_kit_client/examples/test_app.rs
@@ -7,7 +7,7 @@ use gpui::{
 use live_kit_client::{
     capture_local_audio_track, capture_local_video_track,
     id::{ParticipantIdentity, TrackSid},
-    options::TrackPublishOptions,
+    options::{TrackPublishOptions, VideoCodec},
     participant::{Participant, RemoteParticipant},
     play_remote_audio_track,
     publication::LocalTrackPublication,
@@ -287,6 +287,7 @@ impl LivekitWindow {
                         LocalTrack::Video(track),
                         TrackPublishOptions {
                             source: TrackSource::Screenshare,
+                            video_codec: VideoCodec::H264,
                             ..Default::default()
                         },
                     )

diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs
@@ -10,7 +10,6 @@ use cpal::{
 };
 use futures::{Stream, StreamExt as _};
 use gpui::{AppContext, ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream, Task};
-use livekit::webrtc::video_frame::{I420Buffer, VideoBuffer};
 use media::core_video::{CVImageBuffer, CVImageBufferRef};
 use parking_lot::Mutex;
 use std::{borrow::Cow, sync::Arc};

diff --git a/crates/live_kit_client/src/test/webrtc.rs b/crates/live_kit_client/src/test/webrtc.rs
@@ -121,7 +121,7 @@ pub mod video_source {
                 Self { resolution }
             }
 
-            pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, frame: &VideoFrame<T>) {}
+            pub fn capture_frame<T: AsRef<dyn VideoBuffer>>(&self, _frame: &VideoFrame<T>) {}
         }
     }