fix, mac hwcodec decoding align use dst_align (#8215)

Signed-off-by: 21pages <sunboeasy@gmail.com>

libs/scrap/src/common/convert.rs

@@ -13,186 +13,6 @@ use hbb_common::{bail, log, ResultType};
 
 generate_call_macro!(call_yuv, false);
 
-#[cfg(feature = "hwcodec")]
-pub mod hw {
-    use super::*;
-    use crate::ImageFormat;
-    #[cfg(target_os = "windows")]
-    use hwcodec::{ffmpeg::AVPixelFormat, ffmpeg_ram::ffmpeg_linesize_offset_length};
-
-    #[cfg(target_os = "windows")]
-    pub fn hw_nv12_to(
-        fmt: ImageFormat,
-        width: usize,
-        height: usize,
-        src_y: &[u8],
-        src_uv: &[u8],
-        src_stride_y: usize,
-        src_stride_uv: usize,
-        dst: &mut Vec<u8>,
-        i420: &mut Vec<u8>,
-        align: usize,
-    ) -> ResultType<()> {
-        let nv12_stride_y = src_stride_y;
-        let nv12_stride_uv = src_stride_uv;
-        if let Ok((linesize_i420, offset_i420, i420_len)) =
-            ffmpeg_linesize_offset_length(AVPixelFormat::AV_PIX_FMT_YUV420P, width, height, align)
-        {
-            dst.resize(width * height * 4, 0);
-            let i420_stride_y = linesize_i420[0];
-            let i420_stride_u = linesize_i420[1];
-            let i420_stride_v = linesize_i420[2];
-            i420.resize(i420_len as _, 0);
-
-            let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ };
-            let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ };
-            let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ };
-            call_yuv!(NV12ToI420(
-                src_y.as_ptr(),
-                nv12_stride_y as _,
-                src_uv.as_ptr(),
-                nv12_stride_uv as _,
-                i420_offset_y,
-                i420_stride_y,
-                i420_offset_u,
-                i420_stride_u,
-                i420_offset_v,
-                i420_stride_v,
-                width as _,
-                height as _,
-            ));
-            match fmt {
-                ImageFormat::ARGB => {
-                    call_yuv!(I420ToARGB(
-                        i420_offset_y,
-                        i420_stride_y,
-                        i420_offset_u,
-                        i420_stride_u,
-                        i420_offset_v,
-                        i420_stride_v,
-                        dst.as_mut_ptr(),
-                        (width * 4) as _,
-                        width as _,
-                        height as _,
-                    ));
-                }
-                ImageFormat::ABGR => {
-                    call_yuv!(I420ToABGR(
-                        i420_offset_y,
-                        i420_stride_y,
-                        i420_offset_u,
-                        i420_stride_u,
-                        i420_offset_v,
-                        i420_stride_v,
-                        dst.as_mut_ptr(),
-                        (width * 4) as _,
-                        width as _,
-                        height as _,
-                    ));
-                }
-                _ => {
-                    bail!("unsupported image format");
-                }
-            }
-            return Ok(());
-        }
-        bail!("get linesize offset failed");
-    }
-
-    #[cfg(not(target_os = "windows"))]
-    pub fn hw_nv12_to(
-        fmt: ImageFormat,
-        width: usize,
-        height: usize,
-        src_y: &[u8],
-        src_uv: &[u8],
-        src_stride_y: usize,
-        src_stride_uv: usize,
-        dst: &mut Vec<u8>,
-        _i420: &mut Vec<u8>,
-        _align: usize,
-    ) -> ResultType<()> {
-        dst.resize(width * height * 4, 0);
-        match fmt {
-            ImageFormat::ARGB => {
-                call_yuv!(NV12ToARGB(
-                    src_y.as_ptr(),
-                    src_stride_y as _,
-                    src_uv.as_ptr(),
-                    src_stride_uv as _,
-                    dst.as_mut_ptr(),
-                    (width * 4) as _,
-                    width as _,
-                    height as _,
-                ));
-            }
-            ImageFormat::ABGR => {
-                call_yuv!(NV12ToABGR(
-                    src_y.as_ptr(),
-                    src_stride_y as _,
-                    src_uv.as_ptr(),
-                    src_stride_uv as _,
-                    dst.as_mut_ptr(),
-                    (width * 4) as _,
-                    width as _,
-                    height as _,
-                ));
-            }
-            _ => bail!("unsupported image format"),
-        }
-        Ok(())
-    }
-
-    pub fn hw_i420_to(
-        fmt: ImageFormat,
-        width: usize,
-        height: usize,
-        src_y: &[u8],
-        src_u: &[u8],
-        src_v: &[u8],
-        src_stride_y: usize,
-        src_stride_u: usize,
-        src_stride_v: usize,
-        dst: &mut Vec<u8>,
-    ) -> ResultType<()> {
-        let src_y = src_y.as_ptr();
-        let src_u = src_u.as_ptr();
-        let src_v = src_v.as_ptr();
-        dst.resize(width * height * 4, 0);
-        match fmt {
-            ImageFormat::ARGB => {
-                call_yuv!(I420ToARGB(
-                    src_y,
-                    src_stride_y as _,
-                    src_u,
-                    src_stride_u as _,
-                    src_v,
-                    src_stride_v as _,
-                    dst.as_mut_ptr(),
-                    (width * 4) as _,
-                    width as _,
-                    height as _,
-                ));
-            }
-            ImageFormat::ABGR => {
-                call_yuv!(I420ToABGR(
-                    src_y,
-                    src_stride_y as _,
-                    src_u,
-                    src_stride_u as _,
-                    src_v,
-                    src_stride_v as _,
-                    dst.as_mut_ptr(),
-                    (width * 4) as _,
-                    width as _,
-                    height as _,
-                ));
-            }
-            _ => bail!("unsupported image format"),
-        };
-        Ok(())
-    }
-}
 #[cfg(not(target_os = "ios"))]
 pub fn convert_to_yuv(
     captured: &PixelBuffer,

libs/scrap/src/common/hwcodec.rs

@@ -2,7 +2,8 @@ use crate::{
     codec::{
         base_bitrate, codec_thread_num, enable_hwcodec_option, EncoderApi, EncoderCfg, Quality as Q,
     },
-    hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
+    convert::*,
+    CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
     anyhow::{anyhow, bail, Context},
@@ -23,7 +24,7 @@ use hwcodec::{
     ffmpeg_ram::{
         decode::{DecodeContext, DecodeFrame, Decoder},
         encode::{EncodeContext, EncodeFrame, Encoder},
-        CodecInfo,
+        ffmpeg_linesize_offset_length, CodecInfo,
     },
 };
 
@@ -32,6 +33,8 @@ pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30];
 const DEFAULT_GOP: i32 = i32::MAX;
 const DEFAULT_HW_QUALITY: Quality = Quality_Default;
 
+crate::generate_call_macro!(call_yuv, false);
+
 #[derive(Debug, Clone)]
 pub struct HwRamEncoderConfig {
     pub name: String,
@@ -237,9 +240,9 @@ impl HwRamEncoder {
         }
     }
 
-    fn rate_control(config: &HwRamEncoderConfig) -> RateControl {
+    fn rate_control(_config: &HwRamEncoderConfig) -> RateControl {
         #[cfg(target_os = "android")]
-        if config.name.contains("mediacodec") {
+        if _config.name.contains("mediacodec") {
             return RC_VBR;
         }
         RC_CBR
@@ -262,15 +265,15 @@ impl HwRamEncoder {
         quality * factor
     }
 
-    pub fn check_bitrate_range(config: &HwRamEncoderConfig, bitrate: u32) -> u32 {
+    pub fn check_bitrate_range(_config: &HwRamEncoderConfig, bitrate: u32) -> u32 {
         #[cfg(target_os = "android")]
-        if config.name.contains("mediacodec") {
+        if _config.name.contains("mediacodec") {
             let info = crate::android::ffi::get_codec_info();
             if let Some(info) = info {
                 if let Some(codec) = info
                     .codecs
                     .iter()
-                    .find(|c| Some(c.name.clone()) == config.mc_name && c.is_encoder)
+                    .find(|c| Some(c.name.clone()) == _config.mc_name && c.is_encoder)
                 {
                     if codec.max_bitrate > codec.min_bitrate {
                         if bitrate > codec.max_bitrate {
@@ -368,54 +371,100 @@ impl HwRamDecoderImage<'_> {
     // rgb [in/out] fmt and stride must be set in ImageRgb
     pub fn to_fmt(&self, rgb: &mut ImageRgb, i420: &mut Vec<u8>) -> ResultType<()> {
         let frame = self.frame;
-        rgb.w = frame.width as _;
+        let width = frame.width;
-        rgb.h = frame.height as _;
+        let height = frame.height;
-        // take dst_stride into account when you convert
+        rgb.w = width as _;
-        let dst_stride = rgb.stride();
+        rgb.h = height as _;
+        let dst_align = rgb.align();
+        let bytes_per_row = (rgb.w * 4 + dst_align - 1) & !(dst_align - 1);
+        rgb.raw.resize(rgb.h * bytes_per_row, 0);
         match frame.pixfmt {
-            AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to(
+            AVPixelFormat::AV_PIX_FMT_NV12 => {
-                rgb.fmt(),
+                // I420ToARGB is much faster than NV12ToARGB in tests on Windows
-                frame.width as _,
+                if cfg!(windows) {
-                frame.height as _,
+                    let Ok((linesize_i420, offset_i420, len_i420)) = ffmpeg_linesize_offset_length(
-                &frame.data[0],
+                        AVPixelFormat::AV_PIX_FMT_YUV420P,
-                &frame.data[1],
+                        width as _,
-                frame.linesize[0] as _,
+                        height as _,
-                frame.linesize[1] as _,
-                &mut rgb.raw as _,
-                i420,
                         HW_STRIDE_ALIGN,
-            )?,
+                    ) else {
+                        bail!("failed to get i420 linesize, offset, length");
+                    };
+                    i420.resize(len_i420 as _, 0);
+                    let i420_offset_y = unsafe { i420.as_ptr().add(0) as _ };
+                    let i420_offset_u = unsafe { i420.as_ptr().add(offset_i420[0] as _) as _ };
+                    let i420_offset_v = unsafe { i420.as_ptr().add(offset_i420[1] as _) as _ };
+                    call_yuv!(NV12ToI420(
+                        frame.data[0].as_ptr(),
+                        frame.linesize[0],
+                        frame.data[1].as_ptr(),
+                        frame.linesize[1],
+                        i420_offset_y,
+                        linesize_i420[0],
+                        i420_offset_u,
+                        linesize_i420[1],
+                        i420_offset_v,
+                        linesize_i420[2],
+                        width,
+                        height,
+                    ));
+                    let f = match rgb.fmt() {
+                        ImageFormat::ARGB => I420ToARGB,
+                        ImageFormat::ABGR => I420ToABGR,
+                        _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
+                    };
+                    call_yuv!(f(
+                        i420_offset_y,
+                        linesize_i420[0],
+                        i420_offset_u,
+                        linesize_i420[1],
+                        i420_offset_v,
+                        linesize_i420[2],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        width,
+                        height,
+                    ));
+                } else {
+                    let f = match rgb.fmt() {
+                        ImageFormat::ARGB => NV12ToARGB,
+                        ImageFormat::ABGR => NV12ToABGR,
+                        _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
+                    };
+                    call_yuv!(f(
+                        frame.data[0].as_ptr(),
+                        frame.linesize[0],
+                        frame.data[1].as_ptr(),
+                        frame.linesize[1],
+                        rgb.raw.as_mut_ptr(),
+                        bytes_per_row as _,
+                        width,
+                        height,
+                    ));
+                }
+            }
             AVPixelFormat::AV_PIX_FMT_YUV420P => {
-                hw::hw_i420_to(
+                let f = match rgb.fmt() {
-                    rgb.fmt(),
+                    ImageFormat::ARGB => I420ToARGB,
-                    frame.width as _,
+                    ImageFormat::ABGR => I420ToABGR,
-                    frame.height as _,
+                    _ => bail!("unsupported format: {:?} -> {:?}", frame.pixfmt, rgb.fmt()),
-                    &frame.data[0],
+                };
-                    &frame.data[1],
+                call_yuv!(f(
-                    &frame.data[2],
+                    frame.data[0].as_ptr(),
-                    frame.linesize[0] as _,
+                    frame.linesize[0],
-                    frame.linesize[1] as _,
+                    frame.data[1].as_ptr(),
-                    frame.linesize[2] as _,
+                    frame.linesize[1],
-                    &mut rgb.raw as _,
+                    frame.data[2].as_ptr(),
-                )?;
+                    frame.linesize[2],
+                    rgb.raw.as_mut_ptr(),
+                    bytes_per_row as _,
+                    width,
+                    height,
+                ));
             }
         }
         Ok(())
     }
-
-    pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
-        let mut rgb = ImageRgb::new(ImageFormat::ARGB, 1);
-        self.to_fmt(&mut rgb, i420)?;
-        *bgra = rgb.raw;
-        Ok(())
-    }
-
-    pub fn rgba(&self, rgba: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
-        let mut rgb = ImageRgb::new(ImageFormat::ABGR, 1);
-        self.to_fmt(&mut rgb, i420)?;
-        *rgba = rgb.raw;
-        Ok(())
-    }
 }
 
 #[derive(Debug, Eq, PartialEq, Clone, Serialize, Deserialize)]

libs/scrap/src/common/mod.rs

@@ -53,7 +53,7 @@ pub mod record;
 mod vpx;
 
 #[repr(usize)]
-#[derive(Copy, Clone)]
+#[derive(Debug, Copy, Clone)]
 pub enum ImageFormat {
     Raw,
     ABGR,
@@ -65,17 +65,17 @@ pub struct ImageRgb {
     pub w: usize,
     pub h: usize,
     pub fmt: ImageFormat,
-    pub stride: usize,
+    pub align: usize,
 }
 
 impl ImageRgb {
-    pub fn new(fmt: ImageFormat, stride: usize) -> Self {
+    pub fn new(fmt: ImageFormat, align: usize) -> Self {
         Self {
             raw: Vec::new(),
             w: 0,
             h: 0,
             fmt,
-            stride,
+            align,
         }
     }
 
@@ -85,13 +85,13 @@ impl ImageRgb {
     }
 
     #[inline]
-    pub fn stride(&self) -> usize {
+    pub fn align(&self) -> usize {
-        self.stride
+        self.align
     }
 
     #[inline]
-    pub fn set_stride(&mut self, stride: usize) {
+    pub fn set_align(&mut self, align: usize) {
-        self.stride = stride;
+        self.align = align;
     }
 }
 
@@ -378,20 +378,20 @@ pub trait GoogleImage {
     fn stride(&self) -> Vec<i32>;
     fn planes(&self) -> Vec<*mut u8>;
     fn chroma(&self) -> Chroma;
-    fn get_bytes_per_row(w: usize, fmt: ImageFormat, stride: usize) -> usize {
+    fn get_bytes_per_row(w: usize, fmt: ImageFormat, align: usize) -> usize {
         let bytes_per_pixel = match fmt {
             ImageFormat::Raw => 3,
             ImageFormat::ARGB | ImageFormat::ABGR => 4,
         };
         // https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L128
         // https://github.com/lemenkov/libyuv/blob/6900494d90ae095d44405cd4cc3f346971fa69c9/source/convert_argb.cc#L129
-        (w * bytes_per_pixel + stride - 1) & !(stride - 1)
+        (w * bytes_per_pixel + align - 1) & !(align - 1)
     }
     // rgb [in/out] fmt and stride must be set in ImageRgb
     fn to(&self, rgb: &mut ImageRgb) {
         rgb.w = self.width();
         rgb.h = self.height();
-        let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.stride());
+        let bytes_per_row = Self::get_bytes_per_row(rgb.w, rgb.fmt, rgb.align());
         rgb.raw.resize(rgb.h * bytes_per_row, 0);
         let stride = self.stride();
         let planes = self.planes();

src/client.rs

@@ -1052,7 +1052,7 @@ impl VideoHandler {
         log::info!("new video handler for display #{_display}, format: {format:?}, luid: {luid:?}");
         VideoHandler {
             decoder: Decoder::new(format, luid),
-            rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_stride_rgba()),
+            rgb: ImageRgb::new(ImageFormat::ARGB, crate::get_dst_align_rgba()),
             texture: std::ptr::null_mut(),
             recorder: Default::default(),
             record: false,
@@ -1105,7 +1105,7 @@ impl VideoHandler {
     /// Reset the decoder, change format if it is Some
     pub fn reset(&mut self, format: Option<CodecFormat>) {
         #[cfg(target_os = "macos")]
-        self.rgb.set_stride(crate::get_dst_stride_rgba());
+        self.rgb.set_align(crate::get_dst_align_rgba());
         let luid = Self::get_adapter_luid();
         let format = format.unwrap_or(self.decoder.format());
         self.decoder = Decoder::new(format, luid);

src/common.rs

@@ -1617,7 +1617,7 @@ fn read_custom_client_advanced_settings(
 
 #[inline]
 #[cfg(target_os = "macos")]
-pub fn get_dst_stride_rgba() -> usize {
+pub fn get_dst_align_rgba() -> usize {
     // https://developer.apple.com/forums/thread/712709
     // Memory alignment should be multiple of 64.
     if crate::ui_interface::use_texture_render() {
@@ -1629,7 +1629,7 @@ pub fn get_dst_stride_rgba() -> usize {
 
 #[inline]
 #[cfg(not(target_os = "macos"))]
-pub fn get_dst_stride_rgba() -> usize {
+pub fn get_dst_align_rgba() -> usize {
     1
 }
 

src/flutter.rs

@@ -445,7 +445,7 @@ impl VideoRenderer {
                     rgba.raw.len() as _,
                     rgba.w as _,
                     rgba.h as _,
-                    rgba.stride() as _,
+                    rgba.align() as _,
                 )
             };
         }

src/flutter_ffi.rs

@@ -3,7 +3,6 @@ use crate::{
     common::{is_keyboard_mode_supported, make_fd_to_json},
     flutter::{
         self, session_add, session_add_existed, session_start_, sessions, try_sync_peer_option,
-        FlutterHandler,
     },
     input::*,
     ui_interface::{self, *},