From 7b1e361c0ff14d7edb1cbb7951bbe1767d2f4d53 Mon Sep 17 00:00:00 2001 From: Gerwin van der Lugt Date: Thu, 14 Mar 2024 15:19:58 +0100 Subject: [PATCH] starting point for implementing hwaccel --- src/decode.rs | 22 ++++++++ src/error.rs | 10 ++++ src/ffi_hwaccel.rs | 110 ++++++++++++++++++++++++++++++++++++ src/hwaccel.rs | 138 +++++++++++++++++++++++++++++++++++++++++++++ src/lib.rs | 3 + 5 files changed, 283 insertions(+) create mode 100644 src/ffi_hwaccel.rs create mode 100644 src/hwaccel.rs diff --git a/src/decode.rs b/src/decode.rs index 3d15dbf..d99998d 100644 --- a/src/decode.rs +++ b/src/decode.rs @@ -303,6 +303,21 @@ impl DecoderSplit { match self.decoder_receive_frame()? { Some(frame) => { + // TODO: if frame.format() == hw_pix_fmt + let frame = if frame.format() == todo!() { + // Copy frame from hardware acceleration device to host. + + let mut frame_host = RawFrame::empty(); + // TODO: ffi this + // if ((ret = av_hwframe_transfer_data(sw_frame, frame, 0)) < 0) { + // fprintf(stderr, "Error transferring the data to system memory\n"); + // goto fail; + // } + frame_host + } else { + frame + }; + let mut frame_scaled = RawFrame::empty(); self.scaler .run(&frame, &mut frame_scaled) @@ -348,6 +363,10 @@ impl DecoderSplit { let mut decoder = AvContext::new(); set_decoder_context_time_base(&mut decoder, reader_stream.time_base()); decoder.set_parameters(reader_stream.parameters())?; + + // TODO: let hardware_acceleration_context = crate::hwaccel::HardwareAccelerationContext::new(&mut decoder, todo!()); + // TODO: self.hardare_acceleration_context = Some(todo!()); + let decoder = decoder.decoder().video()?; let decoder_time_base = decoder.time_base(); @@ -411,6 +430,9 @@ impl Drop for DecoderSplit { } } } + + // TODO: How are we going to do this? + // TODO: av_buffer_unref(&hw_device_ctx); } } diff --git a/src/error.rs b/src/error.rs index d3f9645..0943b89 100644 --- a/src/error.rs +++ b/src/error.rs @@ -13,6 +13,8 @@ pub enum Error { MissingCodecParameters, UnsupportedCodecParameterSets, InvalidResizeParameters, + UninitializedCodec, + UnsupportedCodecHardwareAccelerationDeviceType, BackendError(FfmpegError), } @@ -26,6 +28,8 @@ impl std::error::Error for Error { Error::MissingCodecParameters => None, Error::UnsupportedCodecParameterSets => None, Error::InvalidResizeParameters => None, + Error::UninitializedCodec => None, + Error::UnsupportedCodecHardwareAccelerationDeviceType => None, Error::BackendError(ref internal) => Some(internal), } } @@ -51,6 +55,12 @@ impl std::fmt::Display for Error { Error::InvalidResizeParameters => { write!(f, "cannot resize frame into provided dimensions") } + Error::UninitializedCodec => { + write!(f, "codec context is not initialized properly") + } + Error::UnsupportedCodecHardwareAccelerationDeviceType => { + write!(f, "codec does not supported hardware acceleration device") + } Error::BackendError(ref internal) => internal.fmt(f), } } diff --git a/src/ffi_hwaccel.rs b/src/ffi_hwaccel.rs new file mode 100644 index 0000000..9769e57 --- /dev/null +++ b/src/ffi_hwaccel.rs @@ -0,0 +1,110 @@ +extern crate ffmpeg_next as ffmpeg; + +use crate::hwaccel::HardwareAccelerationDeviceType; + +pub struct HardwareDeviceContext { + ptr: *mut ffmpeg::ffi::AVBufferRef, +} + +impl HardwareDeviceContext { + pub fn new( + device_type: HardwareAccelerationDeviceType, + ) -> Result { + let mut ptr: *mut ffmpeg::ffi::AVBufferRef = std::ptr::null_mut(); + + unsafe { + match ffmpeg::ffi::av_hwdevice_ctx_create( + (&mut ptr) as *mut *mut ffmpeg::ffi::AVBufferRef, + device_type.into(), + std::ptr::null(), + std::ptr::null_mut(), + 0, + ) { + 0 => Ok(HardwareDeviceContext { ptr }), + e => Err(ffmpeg::error::Error::from(e)), + } + } + } + + unsafe fn ref_raw(&self) -> *mut ffmpeg::ffi::AVBufferRef { + ffmpeg::ffi::av_buffer_ref(self.ptr) + } +} + +impl Drop for HardwareDeviceContext { + fn drop(&mut self) { + unsafe { + ffmpeg::ffi::av_buffer_unref(&mut self.ptr); + } + } +} + +pub fn hwdevice_list_available_device_types() -> Vec { + let mut hwdevice_types = Vec::new(); + let mut hwdevice_type = unsafe { + ffmpeg::ffi::av_hwdevice_iterate_types(ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE) + }; + while hwdevice_type != ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE { + hwdevice_types.push(HardwareAccelerationDeviceType::from(hwdevice_type).unwrap()); + hwdevice_type = unsafe { ffmpeg::ffi::av_hwdevice_iterate_types(hwdevice_type) }; + } + hwdevice_types +} + +pub fn codec_find_corresponding_hwaccel_pixfmt( + codec: &ffmpeg::codec::codec::Codec, + hwaccel_type: HardwareAccelerationDeviceType, +) -> Option { + let mut i = 0; + loop { + unsafe { + let hw_config = ffmpeg::ffi::avcodec_get_hw_config(codec.as_ptr(), i); + if !hw_config.is_null() { + let hw_config_supports_codec = (((*hw_config).methods) as i32 + & ffmpeg::ffi::AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX as i32) + != 0; + if hw_config_supports_codec && (*hw_config).device_type == hwaccel_type.into() { + break Some((*hw_config).pix_fmt.into()); + } + } else { + break None; + } + } + i += 1; + } +} + +pub fn codec_context_hwaccel_set_get_format( + codec_context: &mut ffmpeg::codec::context::Context, + hw_pixfmt: ffmpeg::format::pixel::Pixel, +) { + unsafe { + (*codec_context.as_mut_ptr()).opaque = + ffmpeg::ffi::AVPixelFormat::from(hw_pixfmt) as i32 as _; + (*codec_context.as_mut_ptr()).get_format = Some(hwaccel_get_format); + } +} + +pub fn codec_context_hwaccel_set_hw_device_ctx( + codec_context: &mut ffmpeg::codec::context::Context, + hardware_device_context: &HardwareDeviceContext, +) { + unsafe { + (*codec_context.as_mut_ptr()).hw_device_ctx = hardware_device_context.ref_raw(); + } +} + +#[no_mangle] +unsafe extern "C" fn hwaccel_get_format( + ctx: *mut ffmpeg::ffi::AVCodecContext, + pix_fmts: *const ffmpeg::ffi::AVPixelFormat, +) -> ffmpeg::ffi::AVPixelFormat { + let mut p = pix_fmts; + while *p != ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE { + if *p == std::mem::transmute((*ctx).opaque as i32) { + return *p; + } + p = p.add(1); + } + ffmpeg::ffi::AVPixelFormat::AV_PIX_FMT_NONE +} diff --git a/src/hwaccel.rs b/src/hwaccel.rs new file mode 100644 index 0000000..af374af --- /dev/null +++ b/src/hwaccel.rs @@ -0,0 +1,138 @@ +extern crate ffmpeg_next as ffmpeg; + +use crate::error::Error; +use crate::ffi_hwaccel; + +type Result = std::result::Result; + +pub struct HardwareAccelerationContext { + hardware_device_context: ffi_hwaccel::HardwareDeviceContext, + pixel_format: ffmpeg::util::format::Pixel, +} + +impl HardwareAccelerationContext { + pub fn new( + decoder: &mut ffmpeg::codec::Context, + device_type: HardwareAccelerationDeviceType, + ) -> Result { + let codec = decoder.codec().ok_or(Error::UninitializedCodec)?; + let pixel_format = + ffi_hwaccel::codec_find_corresponding_hwaccel_pixfmt(&codec, device_type) + .ok_or(Error::UnsupportedCodecHardwareAccelerationDeviceType)?; + + ffi_hwaccel::codec_context_hwaccel_set_get_format(decoder, pixel_format); + + let hardware_device_context = ffi_hwaccel::HardwareDeviceContext::new(device_type)?; + ffi_hwaccel::codec_context_hwaccel_set_hw_device_ctx(decoder, &hardware_device_context); + + Ok(HardwareAccelerationContext { + hardware_device_context, + pixel_format, + }) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum HardwareAccelerationDeviceType { + /// Video Decode and Presentation API for Unix (VDPAU) + Vdpau, + /// NVIDIA CUDA + Cuda, + /// Video Acceleration API (VA-API) + VaApi, + /// DirectX Video Acceleration 2.0 + Dxva2, + /// Quick Sync Video + Qsv, + /// VideoToolbox + VideoToolbox, + /// Direct3D 11 Video Acceleration + D3D11Va, + /// Linux Direct Rendering Manager + Drm, + /// OpenCL + OpenCl, + /// MediaCodec + MeiaCodec, + /// Vulkan + Vulkan, + /// Direct3D 12 Video Acceleration + D3D12Va, +} + +impl HardwareAccelerationDeviceType { + /// Whether or not the device type is available on this system. + pub fn is_available(self) -> bool { + Self::list_available().contains(&self) + } + + /// List available hardware acceleration device types on this system. + /// + /// Uses `av_hwdevice_iterate_types` internally. + pub fn list_available() -> Vec { + ffi_hwaccel::hwdevice_list_available_device_types() + } +} + +impl HardwareAccelerationDeviceType { + pub fn from(value: ffmpeg::ffi::AVHWDeviceType) -> Option { + match value { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VDPAU => Some(Self::Vdpau), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA => Some(Self::Cuda), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI => Some(Self::VaApi), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2 => Some(Self::Dxva2), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_QSV => Some(Self::Qsv), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX => Some(Self::VideoToolbox), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA => Some(Self::D3D11Va), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_DRM => Some(Self::Drm), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_OPENCL => Some(Self::OpenCl), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_MEDIACODEC => Some(Self::MeiaCodec), + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN => Some(Self::Vulkan), + // ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_D3D12VA => Self::D3D12Va, + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE => None, + } + } +} + +impl From for ffmpeg::ffi::AVHWDeviceType { + fn from(value: HardwareAccelerationDeviceType) -> Self { + match value { + HardwareAccelerationDeviceType::Vdpau => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VDPAU + } + HardwareAccelerationDeviceType::Cuda => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_CUDA + } + HardwareAccelerationDeviceType::VaApi => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VAAPI + } + HardwareAccelerationDeviceType::Dxva2 => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_DXVA2 + } + HardwareAccelerationDeviceType::Qsv => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_QSV + } + HardwareAccelerationDeviceType::VideoToolbox => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VIDEOTOOLBOX + } + HardwareAccelerationDeviceType::D3D11Va => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_D3D11VA + } + HardwareAccelerationDeviceType::Drm => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_DRM + } + HardwareAccelerationDeviceType::OpenCl => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_OPENCL + } + HardwareAccelerationDeviceType::MeiaCodec => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_MEDIACODEC + } + HardwareAccelerationDeviceType::Vulkan => { + ffmpeg::ffi::AVHWDeviceType::AV_HWDEVICE_TYPE_VULKAN + } + HardwareAccelerationDeviceType::D3D12Va => { + unimplemented!() + } + } + } +} diff --git a/src/lib.rs b/src/lib.rs index 46aed92..3ab9128 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,9 @@ mod encode; mod error; mod extradata; mod ffi; +mod ffi_hwaccel; mod frame; +mod hwaccel; mod init; mod io; mod mux; @@ -20,6 +22,7 @@ pub use error::Error; pub use extradata::{Pps, Sps}; pub use frame::PixelFormat; pub use frame::RawFrame; +pub use hwaccel::HardwareAccelerationDeviceType; pub use init::init; pub use io::{Buf, Reader, Write, Writer}; pub use io::{Locator, Url};