better documentation for the pixel format

Fix bugs in screen share stream configuration

crates/call/Cargo.toml

@@ -42,10 +42,10 @@ serde_derive.workspace = true
 settings.workspace = true
 util.workspace = true
 
-[target.'cfg(target_os = "macos")'.dependencies]
+[target.'cfg(any())'.dependencies]
 livekit_client_macos = { workspace = true }
 
-[target.'cfg(not(target_os = "macos"))'.dependencies]
+[target.'cfg(all())'.dependencies]
 livekit_client = { workspace = true }
 
 [dev-dependencies]

crates/call/src/call.rs

@@ -1,13 +1,13 @@
 pub mod call_settings;
 
-#[cfg(target_os = "macos")]
+#[cfg(any())]
 mod macos;
 
-#[cfg(target_os = "macos")]
+#[cfg(any())]
 pub use macos::*;
 
-#[cfg(not(target_os = "macos"))]
+#[cfg(all())]
 mod cross_platform;
 
-#[cfg(not(target_os = "macos"))]
+#[cfg(all())]
 pub use cross_platform::*;

crates/call/src/cross_platform/room.rs

@@ -1441,7 +1441,7 @@ impl Room {
             let sources = sources.await??;
             let source = sources.first().ok_or_else(|| anyhow!("no display found"))?;
 
-            let (track, stream) = capture_local_video_track(&**source).await?;
+            let (track, stream) = capture_local_video_track(&**source, None).await?;
 
             let publication = participant
                 .publish_track(

crates/collab/src/tests/integration_tests.rs

@@ -2078,17 +2078,7 @@ async fn test_mute_deafen(
                     audio_tracks_playing: participant
                         .audio_tracks
                         .values()
-                        .map({
-                            #[cfg(target_os = "macos")]
-                            {
-                                |track| track.is_playing()
-                            }
-
-                            #[cfg(not(target_os = "macos"))]
-                            {
-                                |(track, _)| track.rtc_track().enabled()
-                            }
-                        })
+                        .map(|(track, _)| track.rtc_track().enabled())
                         .collect(),
                 })
                 .collect::<Vec<_>>()

crates/gpui/src/platform.rs

@@ -239,7 +239,7 @@ pub trait PlatformDisplay: Send + Sync + Debug {
 /// A source of on-screen video content that can be captured.
 pub trait ScreenCaptureSource {
     /// Returns the video resolution of this source.
-    fn resolution(&self) -> Result<Size<Pixels>>;
+    fn resolution(&self) -> Size<DevicePixels>;
 
     /// Start capture video from this source, invoking the given callback
     /// with each frame.
@@ -253,6 +253,7 @@ pub trait ScreenCaptureSource {
 pub trait ScreenCaptureStream {}
 
 /// A frame of video captured from a screen.
+#[derive(Clone)]
 pub struct ScreenCaptureFrame(pub PlatformScreenCaptureFrame);
 
 /// An opaque identifier for a hardware display

crates/gpui/src/platform/mac/screen_capture.rs

@@ -1,6 +1,6 @@
 use crate::{
     platform::{ScreenCaptureFrame, ScreenCaptureSource, ScreenCaptureStream},
-    px, size, Pixels, Size,
+    size, DevicePixels, Size,
 };
 use anyhow::{anyhow, Result};
 use block::ConcreteBlock;
@@ -9,6 +9,10 @@ use cocoa::{
     foundation::NSArray,
 };
 use core_foundation::base::TCFType;
+use core_graphics::display::{
+    CGDirectDisplayID, CGDisplayCopyDisplayMode, CGDisplayModeGetPixelHeight,
+    CGDisplayModeGetPixelWidth, CGDisplayModeRelease,
+};
 use ctor::ctor;
 use futures::channel::oneshot;
 use media::core_media::{CMSampleBuffer, CMSampleBufferRef};
@@ -25,6 +29,7 @@ use std::{cell::RefCell, ffi::c_void, mem, ptr, rc::Rc};
 #[derive(Clone)]
 pub struct MacScreenCaptureSource {
     sc_display: id,
+    size: Size<DevicePixels>,
 }
 
 pub struct MacScreenCaptureStream {
@@ -43,12 +48,8 @@ const FRAME_CALLBACK_IVAR: &str = "frame_callback";
 const SCStreamOutputTypeScreen: NSInteger = 0;
 
 impl ScreenCaptureSource for MacScreenCaptureSource {
-    fn resolution(&self) -> Result<Size<Pixels>> {
-        unsafe {
-            let width: i64 = msg_send![self.sc_display, width];
-            let height: i64 = msg_send![self.sc_display, height];
-            Ok(size(px(width as f32), px(height as f32)))
-        }
+    fn resolution(&self) -> Size<DevicePixels> {
+        self.size
     }
 
     fn stream(
@@ -61,13 +62,21 @@ impl ScreenCaptureSource for MacScreenCaptureSource {
             let configuration: id = msg_send![class!(SCStreamConfiguration), alloc];
             let delegate: id = msg_send![DELEGATE_CLASS, alloc];
             let output: id = msg_send![OUTPUT_CLASS, alloc];
-
             let excluded_windows = NSArray::array(nil);
             let filter: id = msg_send![filter, initWithDisplay:self.sc_display excludingWindows:excluded_windows];
             let configuration: id = msg_send![configuration, init];
             let delegate: id = msg_send![delegate, init];
             let output: id = msg_send![output, init];
 
+            // ASCII for '420f': https://developer.apple.com/documentation/screencapturekit/scstreamconfiguration/pixelformat?language=objc
+            let format = u32::from_be_bytes([52u8, 50u8, 48u8, 102u8]);
+
+            let _: () = msg_send![configuration, setShowsCursor:YES];
+            let _: () = msg_send![configuration, setWidth:self.size.width];
+            let _: () = msg_send![configuration, setHeight:self.size.height];
+            let _: () = msg_send![configuration, setPixelFormat:format];
+            let _: () = msg_send![configuration, setQueueDepth:5i32];
+
             output.as_mut().unwrap().set_ivar(
                 FRAME_CALLBACK_IVAR,
                 Box::into_raw(Box::new(frame_callback)) as *mut c_void,
@@ -94,6 +103,7 @@ impl ScreenCaptureSource for MacScreenCaptureSource {
                             sc_stream: stream,
                             sc_stream_output: output,
                         };
+
                         Ok(Box::new(stream) as Box<dyn ScreenCaptureStream>)
                     } else {
                         let message: id = msg_send![error, localizedDescription];
@@ -159,8 +169,16 @@ pub(crate) fn get_sources() -> oneshot::Receiver<Result<Vec<Box<dyn ScreenCaptur
                 let mut result = Vec::new();
                 for i in 0..displays.count() {
                     let display = displays.objectAtIndex(i);
+                    let display: id = msg_send![display, retain];
+                    let display_id: CGDirectDisplayID = msg_send![display, displayID];
+                    let display_mode_ref = CGDisplayCopyDisplayMode(display_id);
+                    let width = CGDisplayModeGetPixelWidth(display_mode_ref);
+                    let height = CGDisplayModeGetPixelHeight(display_mode_ref);
+                    CGDisplayModeRelease(display_mode_ref);
+
                     let source = MacScreenCaptureSource {
-                        sc_display: msg_send![display, retain],
+                        sc_display: display,
+                        size: size(DevicePixels(width as i32), DevicePixels(height as i32)),
                     };
                     result.push(Box::new(source) as Box<dyn ScreenCaptureSource>);
                 }

crates/gpui/src/platform/test/platform.rs

@@ -1,7 +1,8 @@
 use crate::{
-    px, size, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, ForegroundExecutor,
-    Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame, ScreenCaptureSource,
-    ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance, WindowParams,
+    size, AnyWindowHandle, BackgroundExecutor, ClipboardItem, CursorStyle, DevicePixels,
+    ForegroundExecutor, Keymap, Platform, PlatformDisplay, PlatformTextSystem, ScreenCaptureFrame,
+    ScreenCaptureSource, ScreenCaptureStream, Task, TestDisplay, TestWindow, WindowAppearance,
+    WindowParams,
 };
 use anyhow::Result;
 use collections::VecDeque;
@@ -46,8 +47,8 @@ pub struct TestScreenCaptureSource {}
 pub struct TestScreenCaptureStream {}
 
 impl ScreenCaptureSource for TestScreenCaptureSource {
-    fn resolution(&self) -> Result<crate::Size<crate::Pixels>> {
-        Ok(size(px(1.), px(1.)))
+    fn resolution(&self) -> crate::Size<crate::DevicePixels> {
+        size(DevicePixels(1), DevicePixels(1))
     }
 
     fn stream(

crates/livekit_client/Cargo.toml

@@ -49,6 +49,7 @@ livekit.workspace = true
 [target.'cfg(target_os = "macos")'.dependencies]
 core-foundation.workspace = true
 coreaudio-rs = "0.12.1"
+media.workspace = true
 
 [dev-dependencies]
 collections = { workspace = true, features = ["test-support"] }

crates/livekit_client/examples/test_app.rs

@@ -3,11 +3,12 @@
 // it causes compile errors.
 #![cfg_attr(target_os = "macos", allow(unused_imports))]
 
+use futures::StreamExt;
 use gpui::{
     actions, bounds, div, point,
     prelude::{FluentBuilder as _, IntoElement},
     px, rgb, size, AsyncAppContext, Bounds, InteractiveElement, KeyBinding, Menu, MenuItem,
-    ParentElement, Pixels, Render, ScreenCaptureStream, SharedString,
+    ParentElement, Pixels, Render, ScreenCaptureFrame, ScreenCaptureStream, SharedString,
     StatefulInteractiveElement as _, Styled, Task, View, ViewContext, VisualContext, WindowBounds,
     WindowHandle, WindowOptions,
 };
@@ -22,6 +23,7 @@ use livekit_client::{
     track::{LocalTrack, RemoteTrack, RemoteVideoTrack, TrackSource},
     AudioStream, RemoteVideoTrackView, Room, RoomEvent, RoomOptions,
 };
+use media::core_video::CVImageBuffer;
 #[cfg(not(target_os = "windows"))]
 use postage::stream::Stream;
 
@@ -108,6 +110,7 @@ struct LivekitWindow {
     screen_share_track: Option<LocalTrackPublication>,
     microphone_stream: Option<AudioStream>,
     screen_share_stream: Option<Box<dyn ScreenCaptureStream>>,
+    latest_self_frame: Option<ScreenCaptureFrame>,
     #[cfg(not(target_os = "windows"))]
     remote_participants: Vec<(ParticipantIdentity, ParticipantState)>,
     _events_task: Task<()>,
@@ -156,6 +159,7 @@ impl LivekitWindow {
                             microphone_stream: None,
                             screen_share_track: None,
                             screen_share_stream: None,
+                            latest_self_frame: None,
                             remote_participants: Vec::new(),
                             _events_task,
                         }
@@ -312,7 +316,25 @@ impl LivekitWindow {
             cx.spawn(|this, mut cx| async move {
                 let sources = sources.await.unwrap()?;
                 let source = sources.into_iter().next().unwrap();
-                let (track, stream) = capture_local_video_track(&*source).await?;
+
+                let (self_stream_tx, mut self_stream_rx) = futures::channel::mpsc::unbounded();
+                let (track, stream) =
+                    capture_local_video_track(&*source, Some(self_stream_tx)).await?;
+
+                cx.spawn({
+                    let this = this.clone();
+                    |mut cx| async move {
+                        while let Some(frame) = self_stream_rx.next().await {
+                            this.update(&mut cx, |this, cx| {
+                                this.latest_self_frame = Some(frame);
+                                cx.notify();
+                            })
+                            .ok();
+                        }
+                    }
+                })
+                .detach();
+
                 let publication = participant
                     .publish_track(
                         LocalTrack::Video(track),
@@ -394,6 +416,11 @@ impl Render for LivekitWindow {
                         .on_click(cx.listener(|this, _, cx| this.toggle_screen_share(cx))),
                 ]),
             )
+            .children(
+                self.latest_self_frame
+                    .as_ref()
+                    .map(|frame| gpui::surface(frame.0.clone()).size_full()),
+            )
             .child(
                 div()
                     .id("remote-participants")
@@ -403,7 +430,7 @@ impl Render for LivekitWindow {
                     .flex_grow()
                     .children(self.remote_participants.iter().map(|(identity, state)| {
                         div()
-                            .h(px(300.0))
+                            .size_full()
                             .flex()
                             .flex_col()
                             .m_2()

crates/livekit_client/src/livekit_client.rs

@@ -142,8 +142,9 @@ pub fn init(
 #[cfg(not(target_os = "windows"))]
 pub async fn capture_local_video_track(
     capture_source: &dyn ScreenCaptureSource,
+    show_capture: Option<futures::channel::mpsc::UnboundedSender<ScreenCaptureFrame>>,
 ) -> Result<(track::LocalVideoTrack, Box<dyn ScreenCaptureStream>)> {
-    let resolution = capture_source.resolution()?;
+    let resolution = capture_source.resolution();
     let track_source = NativeVideoSource::new(VideoResolution {
         width: resolution.width.0 as u32,
         height: resolution.height.0 as u32,
@@ -153,6 +154,10 @@ pub async fn capture_local_video_track(
         .stream({
             let track_source = track_source.clone();
             Box::new(move |frame| {
+                if let Some(show_capture) = show_capture.as_ref() {
+                    show_capture.unbounded_send(frame.clone()).unwrap();
+                }
+
                 if let Some(buffer) = video_frame_buffer_to_webrtc(frame) {
                     track_source.capture_frame(&VideoFrame {
                         rotation: VideoRotation::VideoRotation0,

crates/workspace/src/shared_screen.rs

@@ -1,11 +1,11 @@
-#[cfg(target_os = "macos")]
+#[cfg(any())]
 mod macos;
 
-#[cfg(target_os = "macos")]
+#[cfg(any())]
 pub use macos::*;
 
-#[cfg(not(target_os = "macos"))]
+#[cfg(all())]
 mod cross_platform;
 
-#[cfg(not(target_os = "macos"))]
+#[cfg(all())]
 pub use cross_platform::*;