From 47570a14fa7a41236fe57c1b881236b0fe9ab50f Mon Sep 17 00:00:00 2001 From: Richie McIlroy <33632126+richiemcilroy@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:54:40 +0000 Subject: [PATCH] fix: Camera shader issue for YUYV --- apps/desktop/src-tauri/src/lib.rs | 4 +- .../gpu-converters/src/yuyv_rgba/shader.wgsl | 6 + crates/media/src/data/video.rs | 35 +++ crates/media/src/feeds/camera.rs | 206 +++++++++++++----- 4 files changed, 195 insertions(+), 56 deletions(-) create mode 100644 crates/gpu-converters/src/yuyv_rgba/shader.wgsl create mode 100644 crates/media/src/data/video.rs diff --git a/apps/desktop/src-tauri/src/lib.rs b/apps/desktop/src-tauri/src/lib.rs index 9f450d78..6d6eeea1 100644 --- a/apps/desktop/src-tauri/src/lib.rs +++ b/apps/desktop/src-tauri/src/lib.rs @@ -912,8 +912,8 @@ async fn copy_video_to_clipboard( #[derive(Serialize, Deserialize, specta::Type)] pub struct VideoRecordingMetadata { - duration: f64, - size: f64, + pub duration: f64, + pub size: f64, } #[tauri::command] diff --git a/crates/gpu-converters/src/yuyv_rgba/shader.wgsl b/crates/gpu-converters/src/yuyv_rgba/shader.wgsl new file mode 100644 index 00000000..192050ac --- /dev/null +++ b/crates/gpu-converters/src/yuyv_rgba/shader.wgsl @@ -0,0 +1,6 @@ + // Each texel stores two Y’s => two output RGBA pixels. + let out_coords = gid.xy * vec2(2, 1); + + textureStore(rgba_output, out_coords, vec4(r1, g1, b1, 1.0)); + textureStore(rgba_output, out_coords + vec2(1, 0), vec4(r2, g2, b2, 1.0)); +} \ No newline at end of file diff --git a/crates/media/src/data/video.rs b/crates/media/src/data/video.rs new file mode 100644 index 00000000..d58a4a5e --- /dev/null +++ b/crates/media/src/data/video.rs @@ -0,0 +1,35 @@ +pub struct FFVideo(ffmpeg::frame::Video); + +impl FFVideo { + pub fn new(format: ffmpeg::format::Pixel, width: u32, height: u32) -> Self { + Self(ffmpeg::frame::Video::new(format, width, height)) + } + + pub fn data(&self, index: usize) -> &[u8] { + self.0.data(index) + } + + pub fn data_mut(&mut self, index: usize) -> &mut [u8] { + self.0.data_mut(index) + } + + pub fn stride(&self, index: usize) -> usize { + self.0.stride(index) + } + + pub fn as_mut_ptr(&mut self) -> *mut ffmpeg_sys_next::AVFrame { + self.0.as_mut_ptr() + } + + pub fn as_ptr(&self) -> *const ffmpeg_sys_next::AVFrame { + self.0.as_ptr() + } + + pub fn convert_with( + &mut self, + context: &mut ffmpeg::software::scaling::Context, + output: &mut FFVideo, + ) -> Result<(), ffmpeg::Error> { + unsafe { context.run(&self.0, &mut output.0) } + } +} diff --git a/crates/media/src/feeds/camera.rs b/crates/media/src/feeds/camera.rs index e790d76c..dff46e90 100644 --- a/crates/media/src/feeds/camera.rs +++ b/crates/media/src/feeds/camera.rs @@ -84,7 +84,7 @@ impl CameraFeed { .into_iter() .map(|i| i.human_name().to_string()) .collect::>(), - Err(_) => Vec::new() + Err(_) => Vec::new(), } } @@ -144,7 +144,7 @@ fn create_camera(info: &CameraInfo) -> Result { RequestedFormatType::AbsoluteHighestFrameRate, &[FrameFormat::YUYV], ); - + #[cfg(feature = "debug-logging")] trace!("Requested camera format: {:?}", format); @@ -160,10 +160,10 @@ fn create_camera(info: &CameraInfo) -> Result { } let camera = Camera::new(index, format)?; - + #[cfg(feature = "debug-logging")] debug!("Created camera with format: {:?}", camera.camera_format()); - + Ok(camera) } @@ -226,7 +226,7 @@ fn run_camera_feed( loop { match control.try_recv() { Err(TryRecvError::Disconnected) => break, - Err(TryRecvError::Empty) => {}, + Err(TryRecvError::Empty) => {} Ok(CameraControl::Shutdown) => break, Ok(CameraControl::AttachRawConsumer(rgba_sender)) => { maybe_raw_data = Some(rgba_sender); @@ -364,18 +364,19 @@ impl FrameConverter { let context = ffmpeg::software::converter( (video_info.width, video_info.height), if camera_format.format() == FrameFormat::YUYV { - ffmpeg::format::Pixel::YUYV422 + ffmpeg::format::Pixel::UYVY422 } else { video_info.pixel_format }, ffmpeg::format::Pixel::RGBA, - ).unwrap(); + ) + .unwrap(); Self { video_info, context, format: camera_format.format(), - hw_converter: None, // Don't use hardware converters + hw_converter: None, // Don't use hardware converters } } @@ -383,38 +384,55 @@ impl FrameConverter { let resolution = buffer.resolution(); match self.format { - FrameFormat::YUYV => { - self.convert_with_ffmpeg(buffer, resolution) - } - _ => { - match &self.hw_converter { - Some(HwConverter::NV12(converter)) => { - converter.convert( - NV12Input::from_buffer(buffer.buffer(), resolution.width(), resolution.height()), - resolution.width(), - resolution.height(), - ) - } - _ => { - self.convert_with_ffmpeg(buffer, resolution) - } - } - } + FrameFormat::YUYV => self.convert_with_ffmpeg(buffer, resolution), + _ => match &self.hw_converter { + Some(HwConverter::NV12(converter)) => converter.convert( + NV12Input::from_buffer( + buffer.buffer(), + resolution.width(), + resolution.height(), + ), + resolution.width(), + resolution.height(), + ), + _ => self.convert_with_ffmpeg(buffer, resolution), + }, } } fn convert_with_ffmpeg(&mut self, buffer: &nokhwa::Buffer, resolution: Resolution) -> Vec { if self.format == FrameFormat::YUYV { // For YUYV, we need to handle the conversion differently - let stride = resolution.width() as usize * 2; // YUYV uses 2 bytes per pixel - - // Create input frame with YUYV format - let mut input_frame = FFVideo::new(ffmpeg::format::Pixel::YUYV422, resolution.width(), resolution.height()); - input_frame.data_mut(0).copy_from_slice(buffer.buffer()); - - // Convert directly to RGBA - let mut rgba_frame = FFVideo::new(ffmpeg::format::Pixel::RGBA, resolution.width(), resolution.height()); - + let stride = resolution.width() as usize * 2; // YUYV uses 2 bytes per pixel + let src = buffer.buffer(); + + // Create input frame with YUYV format and copy data + let mut input_frame = FFVideo::new( + ffmpeg::format::Pixel::UYVY422, + resolution.width(), + resolution.height(), + ); + + // Copy data line by line + { + let dst_stride = input_frame.stride(0); + let dst = input_frame.data_mut(0); + for y in 0..resolution.height() as usize { + let src_offset = y * stride; + let dst_offset = y * dst_stride; + dst[dst_offset..dst_offset + stride] + .copy_from_slice(&src[src_offset..src_offset + stride]); + } + } + + // Create output frame + let mut rgba_frame = FFVideo::new( + ffmpeg::format::Pixel::RGBA, + resolution.width(), + resolution.height(), + ); + + // Convert the frame if self.context.run(&input_frame, &mut rgba_frame).is_ok() { rgba_frame.data(0).to_vec() } else { @@ -422,10 +440,50 @@ impl FrameConverter { } } else { // For other formats, use the normal conversion path - let stride = resolution.width() as usize * 4; // RGBA uses 4 bytes per pixel - let input_frame = self.video_info.wrap_frame(buffer.buffer(), 0, stride); - - let mut rgba_frame = FFVideo::empty(); + let stride = match self.format { + FrameFormat::NV12 => resolution.width() as usize, + FrameFormat::BGRA => resolution.width() as usize * 4, + FrameFormat::MJPEG => resolution.width() as usize * 4, + FrameFormat::GRAY => resolution.width() as usize, + FrameFormat::RAWRGB => resolution.width() as usize * 3, + _ => buffer.buffer_bytes().len() / resolution.height() as usize, + }; + + // Create input frame and copy data + let mut input_frame = FFVideo::new( + match self.format { + FrameFormat::NV12 => ffmpeg::format::Pixel::NV12, + FrameFormat::BGRA => ffmpeg::format::Pixel::BGRA, + FrameFormat::MJPEG => ffmpeg::format::Pixel::RGBA, + FrameFormat::GRAY => ffmpeg::format::Pixel::GRAY8, + FrameFormat::RAWRGB => ffmpeg::format::Pixel::RGB24, + _ => ffmpeg::format::Pixel::RGBA, + }, + resolution.width(), + resolution.height(), + ); + + // Copy data line by line + { + let dst_stride = input_frame.stride(0); + let dst = input_frame.data_mut(0); + let src = buffer.buffer(); + for y in 0..resolution.height() as usize { + let src_offset = y * stride; + let dst_offset = y * dst_stride; + dst[dst_offset..dst_offset + stride] + .copy_from_slice(&src[src_offset..src_offset + stride]); + } + } + + // Create output frame + let mut rgba_frame = FFVideo::new( + ffmpeg::format::Pixel::RGBA, + resolution.width(), + resolution.height(), + ); + + // Convert the frame if self.context.run(&input_frame, &mut rgba_frame).is_ok() { rgba_frame.data(0).to_vec() } else { @@ -436,31 +494,71 @@ impl FrameConverter { fn raw(&mut self, buffer: &nokhwa::Buffer) -> FFVideo { let resolution = buffer.resolution(); - + if self.format == FrameFormat::YUYV { // For YUYV, we need to handle the conversion differently - let stride = resolution.width() as usize * 2; // YUYV uses 2 bytes per pixel - - // Create input frame with YUYV format - let mut input_frame = FFVideo::new(ffmpeg::format::Pixel::YUYV422, resolution.width(), resolution.height()); - input_frame.data_mut(0).copy_from_slice(buffer.buffer()); + let stride = resolution.width() as usize * 2; // YUYV uses 2 bytes per pixel + let src = buffer.buffer(); + + // Create input frame with YUYV format and copy data + let mut input_frame = FFVideo::new( + ffmpeg::format::Pixel::UYVY422, + resolution.width(), + resolution.height(), + ); + + // Copy data line by line + { + let dst_stride = input_frame.stride(0); + let dst = input_frame.data_mut(0); + for y in 0..resolution.height() as usize { + let src_offset = y * stride; + let dst_offset = y * dst_stride; + dst[dst_offset..dst_offset + stride] + .copy_from_slice(&src[src_offset..src_offset + stride]); + } + } + input_frame } else { // For other formats, use the normal conversion path let stride = match self.format { - FrameFormat::NV12 => resolution.width() as usize, // 1 byte per pixel for Y plane - FrameFormat::BGRA => resolution.width() as usize * 4, // 4 bytes per pixel - FrameFormat::MJPEG => resolution.width() as usize * 4, // 4 bytes per pixel - FrameFormat::GRAY => resolution.width() as usize, // 1 byte per pixel - FrameFormat::RAWRGB => resolution.width() as usize * 3, // 3 bytes per pixel + FrameFormat::NV12 => resolution.width() as usize, + FrameFormat::BGRA => resolution.width() as usize * 4, + FrameFormat::MJPEG => resolution.width() as usize * 4, + FrameFormat::GRAY => resolution.width() as usize, + FrameFormat::RAWRGB => resolution.width() as usize * 3, _ => buffer.buffer_bytes().len() / resolution.height() as usize, }; - self.video_info.wrap_frame( - buffer.buffer(), - 0, - stride, - ) + // Create input frame and copy data + let mut input_frame = FFVideo::new( + match self.format { + FrameFormat::NV12 => ffmpeg::format::Pixel::NV12, + FrameFormat::BGRA => ffmpeg::format::Pixel::BGRA, + FrameFormat::MJPEG => ffmpeg::format::Pixel::RGBA, + FrameFormat::GRAY => ffmpeg::format::Pixel::GRAY8, + FrameFormat::RAWRGB => ffmpeg::format::Pixel::RGB24, + _ => ffmpeg::format::Pixel::RGBA, + }, + resolution.width(), + resolution.height(), + ); + + // Copy data line by line + { + let dst_stride = input_frame.stride(0); + let dst = input_frame.data_mut(0); + let src = buffer.buffer(); + for y in 0..resolution.height() as usize { + let src_offset = y * stride; + let dst_offset = y * dst_stride; + dst[dst_offset..dst_offset + stride] + .copy_from_slice(&src[src_offset..src_offset + stride]); + } + } + + input_frame } } }