diff --git a/Cargo.lock b/Cargo.lock index 86999c7159..779ede6544 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7131,6 +7131,7 @@ dependencies = [ "gpui", "http 0.2.12", "http_client", + "image", "live_kit_server", "livekit", "log", @@ -7142,6 +7143,7 @@ dependencies = [ "serde_json", "sha2", "simplelog", + "smallvec", "util", ] diff --git a/crates/gpui/src/platform/linux.rs b/crates/gpui/src/platform/linux.rs index 089b52cf1e..08afde4d6d 100644 --- a/crates/gpui/src/platform/linux.rs +++ b/crates/gpui/src/platform/linux.rs @@ -21,4 +21,6 @@ pub(crate) use wayland::*; #[cfg(feature = "x11")] pub(crate) use x11::*; -pub(crate) type PlatformScreenCaptureFrame = (); +// TODO(mgsloan): This type won't make sense for frame capture. A `type VideoFrame` with this type +// should be added to `live_kit_client`. +pub(crate) type PlatformScreenCaptureFrame = std::sync::Arc; diff --git a/crates/gpui/src/platform/windows.rs b/crates/gpui/src/platform/windows.rs index 51d09f0013..933f0255fa 100644 --- a/crates/gpui/src/platform/windows.rs +++ b/crates/gpui/src/platform/windows.rs @@ -22,4 +22,6 @@ pub(crate) use wrapper::*; pub(crate) use windows::Win32::Foundation::HWND; -pub(crate) type PlatformScreenCaptureFrame = (); +// TODO(mgsloan): This type won't make sense for frame capture. A `type VideoFrame` with this type +// should be added to `live_kit_client`. +pub(crate) type PlatformScreenCaptureFrame = std::sync::Arc; diff --git a/crates/live_kit_client/Cargo.toml b/crates/live_kit_client/Cargo.toml index 46454170ba..0c4cf84ca5 100644 --- a/crates/live_kit_client/Cargo.toml +++ b/crates/live_kit_client/Cargo.toml @@ -40,6 +40,8 @@ parking_lot.workspace = true postage.workspace = true util.workspace = true http_client.workspace = true +smallvec.workspace = true +image.workspace = true [target.'cfg(not(target_os = "windows"))'.dependencies] livekit.workspace = true diff --git a/crates/live_kit_client/src/live_kit_client.rs b/crates/live_kit_client/src/live_kit_client.rs index 83cd74a534..98c45fd4da 100644 --- a/crates/live_kit_client/src/live_kit_client.rs +++ b/crates/live_kit_client/src/live_kit_client.rs @@ -460,9 +460,44 @@ fn video_frame_buffer_from_webrtc(buffer: Box) -> Option) -> Option { - None +#[cfg(any(target_os = "linux", target_os = "windows"))] +fn video_frame_buffer_from_webrtc(buffer: Box) -> Option { + use std::alloc::{alloc, Layout}; + + use gpui::RenderImage; + use image::{Frame, RgbaImage}; + use livekit::webrtc::prelude::VideoFormatType; + use smallvec::SmallVec; + + let width = buffer.width(); + let height = buffer.height(); + let stride = width * 4; + let byte_len = (stride * height) as usize; + let bgra_frame_vec = unsafe { + // Motivation for this unsafe code is to avoid initializing the frame data, since to_argb + // will write all bytes anyway. + let start_ptr = alloc(Layout::array::(byte_len).unwrap()); + let bgra_frame_slice = std::slice::from_raw_parts_mut(start_ptr, byte_len); + buffer.to_argb( + VideoFormatType::BGRA, + bgra_frame_slice, + stride, + width as i32, + height as i32, + ); + Vec::from_raw_parts(start_ptr, byte_len, byte_len) + }; + + Some(ScreenCaptureFrame(Arc::new(RenderImage::new( + SmallVec::from_elem( + Frame::new( + RgbaImage::from_raw(width, height, bgra_frame_vec) + .with_context(|| "Bug: not enough bytes allocated for image.") + .unwrap(), + ), + 1, + ), + )))) } #[cfg(target_os = "macos")] diff --git a/crates/live_kit_client/src/remote_video_track_view.rs b/crates/live_kit_client/src/remote_video_track_view.rs index bbfaea1875..cbae09a8f8 100644 --- a/crates/live_kit_client/src/remote_video_track_view.rs +++ b/crates/live_kit_client/src/remote_video_track_view.rs @@ -1,10 +1,14 @@ +use std::sync::Arc; + use crate::track::RemoteVideoTrack; use anyhow::Result; use futures::StreamExt as _; use gpui::{ - Empty, EventEmitter, IntoElement, Render, ScreenCaptureFrame, Task, View, ViewContext, - VisualContext as _, + img, Empty, EventEmitter, IntoElement, Render, RenderImage, ScreenCaptureFrame, Task, View, + ViewContext, VisualContext as _, }; +use image::Frame; +use smallvec::SmallVec; pub struct RemoteVideoTrackView { track: RemoteVideoTrack, @@ -56,6 +60,11 @@ impl Render for RemoteVideoTrackView { .into_any_element(); } + #[cfg(not(target_os = "macos"))] + if let Some(frame) = &self.frame { + return img(frame.0.clone()).into_any_element(); + } + Empty.into_any_element() } }