Initial implementation of rendering screen shares on Linux and Windows #13714

Cargo.lock

@@ -7131,6 +7131,7 @@ dependencies = [
  "gpui",
  "http 0.2.12",
  "http_client",
+ "image",
  "live_kit_server",
  "livekit",
  "log",
@@ -7142,6 +7143,7 @@ dependencies = [
  "serde_json",
  "sha2",
  "simplelog",
+ "smallvec",
  "util",
 ]
 

crates/gpui/src/platform/linux.rs

@@ -21,4 +21,6 @@ pub(crate) use wayland::*;
 #[cfg(feature = "x11")]
 pub(crate) use x11::*;
 
-pub(crate) type PlatformScreenCaptureFrame = ();
+// TODO(mgsloan): This type won't make sense for frame capture. A `type VideoFrame` with this type
+// should be added to `live_kit_client`.
+pub(crate) type PlatformScreenCaptureFrame = std::sync::Arc<crate::RenderImage>;

crates/gpui/src/platform/windows.rs

@@ -22,4 +22,6 @@ pub(crate) use wrapper::*;
 
 pub(crate) use windows::Win32::Foundation::HWND;
 
-pub(crate) type PlatformScreenCaptureFrame = ();
+// TODO(mgsloan): This type won't make sense for frame capture. A `type VideoFrame` with this type
+// should be added to `live_kit_client`.
+pub(crate) type PlatformScreenCaptureFrame = std::sync::Arc<crate::RenderImage>;

crates/live_kit_client/Cargo.toml

@@ -40,6 +40,8 @@ parking_lot.workspace = true
 postage.workspace = true
 util.workspace = true
 http_client.workspace = true
+smallvec.workspace = true
+image.workspace = true
 
 [target.'cfg(not(target_os = "windows"))'.dependencies]
 livekit.workspace = true

crates/live_kit_client/src/live_kit_client.rs

@@ -460,9 +460,44 @@ fn video_frame_buffer_from_webrtc(buffer: Box<dyn VideoBuffer>) -> Option<Screen
     }
 }
 
-#[cfg(not(any(target_os = "macos", target_os = "windows")))]
-fn video_frame_buffer_from_webrtc(_buffer: Box<dyn VideoBuffer>) -> Option<ScreenCaptureFrame> {
-    None
+#[cfg(any(target_os = "linux", target_os = "windows"))]
+fn video_frame_buffer_from_webrtc(buffer: Box<dyn VideoBuffer>) -> Option<ScreenCaptureFrame> {
+    use std::alloc::{alloc, Layout};
+
+    use gpui::RenderImage;
+    use image::{Frame, RgbaImage};
+    use livekit::webrtc::prelude::VideoFormatType;
+    use smallvec::SmallVec;
+
+    let width = buffer.width();
+    let height = buffer.height();
+    let stride = width * 4;
+    let byte_len = (stride * height) as usize;
+    let bgra_frame_vec = unsafe {
+        // Motivation for this unsafe code is to avoid initializing the frame data, since to_argb
+        // will write all bytes anyway.
+        let start_ptr = alloc(Layout::array::<u8>(byte_len).unwrap());
+        let bgra_frame_slice = std::slice::from_raw_parts_mut(start_ptr, byte_len);
+        buffer.to_argb(
+            VideoFormatType::BGRA,
+            bgra_frame_slice,
+            stride,
+            width as i32,
+            height as i32,
+        );
+        Vec::from_raw_parts(start_ptr, byte_len, byte_len)
+    };
+
+    Some(ScreenCaptureFrame(Arc::new(RenderImage::new(
+        SmallVec::from_elem(
+            Frame::new(
+                RgbaImage::from_raw(width, height, bgra_frame_vec)
+                    .with_context(|| "Bug: not enough bytes allocated for image.")
+                    .unwrap(),
+            ),
+            1,
+        ),
+    ))))
 }
 
 #[cfg(target_os = "macos")]

crates/live_kit_client/src/remote_video_track_view.rs

@@ -1,10 +1,14 @@
+use std::sync::Arc;
+
 use crate::track::RemoteVideoTrack;
 use anyhow::Result;
 use futures::StreamExt as _;
 use gpui::{
-    Empty, EventEmitter, IntoElement, Render, ScreenCaptureFrame, Task, View, ViewContext,
-    VisualContext as _,
+    img, Empty, EventEmitter, IntoElement, Render, RenderImage, ScreenCaptureFrame, Task, View,
+    ViewContext, VisualContext as _,
 };
+use image::Frame;
+use smallvec::SmallVec;
 
 pub struct RemoteVideoTrackView {
     track: RemoteVideoTrack,
@@ -56,6 +60,11 @@ impl Render for RemoteVideoTrackView {
                 .into_any_element();
         }
 
+        #[cfg(not(target_os = "macos"))]
+        if let Some(frame) = &self.frame {
+            return img(frame.0.clone()).into_any_element();
+        }
+
         Empty.into_any_element()
     }
 }