Add APIs for converting between native VideoFrameBuffers and their underlying CVPixelBuffer on macOS

libwebrtc/src/native/video_frame.rs

@@ -148,6 +148,24 @@ macro_rules! impl_to_argb {
 
 #[allow(unused_unsafe)]
 impl NativeBuffer {
+    #[cfg(target_os = "macos")]
+    pub unsafe fn from_cv_pixel_buffer(
+        cv_pixel_buffer: *mut std::ffi::c_void,
+    ) -> vf::native::NativeBuffer {
+        vf::native::NativeBuffer {
+            handle: NativeBuffer {
+                sys_handle: vfb_sys::ffi::new_native_buffer_from_platform_image_buffer(
+                    cv_pixel_buffer as *mut _,
+                ),
+            },
+        }
+    }
+
+    #[cfg(target_os = "macos")]
+    pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
+        unsafe { vfb_sys::ffi::native_buffer_to_platform_image_buffer(&self.sys_handle) as *mut _ }
+    }
+
     pub fn sys_handle(&self) -> &vfb_sys::ffi::VideoFrameBuffer {
         &self.sys_handle
     }

libwebrtc/src/native/video_source.rs

@@ -18,6 +18,7 @@ use std::{
 };
 
 use cxx::SharedPtr;
+use livekit_runtime::interval;
 use parking_lot::Mutex;
 use webrtc_sys::{video_frame as vf_sys, video_frame::ffi::VideoRotation, video_track as vt_sys};
 
@@ -61,7 +62,7 @@ impl NativeVideoSource {
             let source = source.clone();
             let i420 = I420Buffer::new(resolution.width, resolution.height);
             async move {
-                let mut interval = tokio::time::interval(Duration::from_millis(100)); // 10 fps
+                let mut interval = interval(Duration::from_millis(100)); // 10 fps
 
                 loop {
                     interval.tick().await;

libwebrtc/src/video_frame.rs

@@ -449,6 +449,27 @@ pub mod native {
 
     new_buffer_type!(NativeBuffer, Native, as_native);
 
+    impl NativeBuffer {
+        /// Creates a `NativeBuffer` from a `CVPixelBufferRef` pointer.
+        ///
+        /// This function does not bump the reference count of the pixel buffer.
+        ///
+        /// Safety: The given pointer must be a valid `CVPixelBufferRef`.
+        #[cfg(target_os = "macos")]
+        pub unsafe fn from_cv_pixel_buffer(cv_pixel_buffer: *mut std::ffi::c_void) -> Self {
+            vf_imp::NativeBuffer::from_cv_pixel_buffer(cv_pixel_buffer)
+        }
+
+        /// Returns the `CVPixelBufferRef` that backs this buffer, or `null` if
+        /// this buffer is not currently backed by a `CVPixelBufferRef`.
+        ///
+        /// This function does not bump the reference count of the pixel buffer.
+        #[cfg(target_os = "macos")]
+        pub fn get_cv_pixel_buffer(&self) -> *mut std::ffi::c_void {
+            self.handle.get_cv_pixel_buffer()
+        }
+    }
+
     pub trait VideoFrameBufferExt: VideoBuffer {
         fn to_i420(&self) -> I420Buffer;
         fn to_argb(

webrtc-sys/build.rs

@@ -152,7 +152,11 @@ fn main() {
 
             configure_darwin_sysroot(&mut builder);
 
-            builder.file("src/objc_video_factory.mm").flag("-stdlib=libc++").flag("-std=c++20");
+            builder
+                .file("src/objc_video_factory.mm")
+                .file("src/objc_video_frame_buffer.mm")
+                .flag("-stdlib=libc++")
+                .flag("-std=c++20");
         }
         "ios" => {
             println!("cargo:rustc-link-lib=framework=CoreFoundation");

webrtc-sys/include/livekit/video_frame_buffer.h

@@ -39,6 +39,18 @@ class I444Buffer;
 class I010Buffer;
 class NV12Buffer;
 }  // namespace livekit
+
+#ifdef __APPLE__
+#include <CoreVideo/CoreVideo.h>
+namespace livekit {
+typedef __CVBuffer PlatformImageBuffer;
+}  // namespace livekit
+#else
+namespace livekit {
+typedef void PlatformImageBuffer;
+}  // namespace livekit
+#endif
+
 #include "webrtc-sys/src/video_frame_buffer.rs.h"
 
 namespace livekit {
@@ -180,6 +192,9 @@ std::unique_ptr<I444Buffer> new_i444_buffer(int width, int height, int stride_y,
 std::unique_ptr<I010Buffer> new_i010_buffer(int width, int height, int stride_y, int stride_u, int stride_v);
 std::unique_ptr<NV12Buffer> new_nv12_buffer(int width, int height, int stride_y, int stride_uv);
 
+std::unique_ptr<VideoFrameBuffer> new_native_buffer_from_platform_image_buffer(PlatformImageBuffer *buffer);
+PlatformImageBuffer* native_buffer_to_platform_image_buffer(const std::unique_ptr<VideoFrameBuffer> &);
+
 static const VideoFrameBuffer* yuv_to_vfb(const PlanarYuvBuffer* yuv) {
   return yuv;
 }

webrtc-sys/src/objc_video_frame_buffer.mm

@@ -0,0 +1,48 @@
+/*
+ * Copyright 2024 LiveKit
+ *
+ * Licensed under the Apache License, Version 2.0 (the “License”);
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an “AS IS” BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "livekit/video_frame_buffer.h"
+
+#import <CoreVideo/CoreVideo.h>
+#import <sdk/objc/components/video_frame_buffer/RTCCVPixelBuffer.h>
+#include "sdk/objc/native/api/video_frame_buffer.h"
+
+namespace livekit {
+
+std::unique_ptr<VideoFrameBuffer> new_native_buffer_from_platform_image_buffer(
+    CVPixelBufferRef pixelBuffer
+) {
+    RTCCVPixelBuffer *buffer = [[RTCCVPixelBuffer alloc] initWithPixelBuffer:pixelBuffer];
+    rtc::scoped_refptr<webrtc::VideoFrameBuffer> frame_buffer = webrtc::ObjCToNativeVideoFrameBuffer(buffer);
+    [buffer release];
+    CVPixelBufferRelease(pixelBuffer);
+    return std::make_unique<VideoFrameBuffer>(frame_buffer);
+}
+
+CVPixelBufferRef native_buffer_to_platform_image_buffer(
+    const std::unique_ptr<VideoFrameBuffer> &buffer
+) {
+    id<RTC_OBJC_TYPE(RTCVideoFrameBuffer)> rtc_pixel_buffer = webrtc::NativeToObjCVideoFrameBuffer(buffer->get());
+
+    if ([rtc_pixel_buffer isKindOfClass:[RTCCVPixelBuffer class]]) {
+        RTCCVPixelBuffer *cv_pixel_buffer = (RTCCVPixelBuffer *)rtc_pixel_buffer;
+        return [cv_pixel_buffer pixelBuffer];
+    } else {
+        return nullptr;
+    }
+}
+
+}  // namespace livekit

webrtc-sys/src/video_frame_buffer.cpp

@@ -269,4 +269,20 @@ std::unique_ptr<NV12Buffer> new_nv12_buffer(int width,
       webrtc::NV12Buffer::Create(width, height, stride_y, stride_uv));
 }
 
+#ifndef __APPLE__
+
+std::unique_ptr<VideoFrameBuffer> new_native_buffer_from_platform_image_buffer(
+    PlatformImageBuffer *buffer
+) {
+  return nullptr;
+}
+
+PlatformImageBuffer* native_buffer_to_platform_image_buffer(
+    const std::unique_ptr<VideoFrameBuffer> &buffer
+) {
+  return nullptr;
+}
+
+#endif
+
 }  // namespace livekit

webrtc-sys/src/video_frame_buffer.rs

@@ -43,6 +43,7 @@ pub mod ffi {
         type I444Buffer;
         type I010Buffer;
         type NV12Buffer;
+        type PlatformImageBuffer;
 
         fn buffer_type(self: &VideoFrameBuffer) -> VideoFrameBufferType;
         fn width(self: &VideoFrameBuffer) -> u32;
@@ -126,6 +127,13 @@ pub mod ffi {
             stride_uv: i32,
         ) -> UniquePtr<NV12Buffer>;
 
+        unsafe fn new_native_buffer_from_platform_image_buffer(
+            platform_native_buffer: *mut PlatformImageBuffer,
+        ) -> UniquePtr<VideoFrameBuffer>;
+        unsafe fn native_buffer_to_platform_image_buffer(
+            buffer: &UniquePtr<VideoFrameBuffer>,
+        ) -> *mut PlatformImageBuffer;
+
         unsafe fn yuv_to_vfb(yuv: *const PlanarYuvBuffer) -> *const VideoFrameBuffer;
         unsafe fn biyuv_to_vfb(yuv: *const BiplanarYuvBuffer) -> *const VideoFrameBuffer;
         unsafe fn yuv8_to_yuv(yuv8: *const PlanarYuv8Buffer) -> *const PlanarYuvBuffer;