android: for software encoding and screen size > 1200, display size reduced to half (#8167)

Signed-off-by: 21pages <sunboeasy@gmail.com>

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -55,6 +55,8 @@ const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9
 
 // video const
 
+const val MAX_SCREEN_SIZE = 1200
+
 const val VIDEO_KEY_BIT_RATE = 1024_000
 const val VIDEO_KEY_FRAME_RATE = 30
 
@@ -170,6 +172,14 @@ class MainService : Service() {
                 Log.d(logTag, "from rust:stop_capture")
                 stopCapture()
             }
+            "is_hardware_codec" -> {
+                val isHwCodec = arg1.toBoolean()
+                if (isHardwareCodec != isHwCodec) {
+                    isHardwareCodec = isHwCodec
+                    updateScreenInfo(resources.configuration.orientation)
+                }
+                
+            }
             else -> {
             }
         }
@@ -245,6 +255,7 @@ class MainService : Service() {
         super.onDestroy()
     }
 
+    private var isHardwareCodec: Boolean? = null;
     private fun updateScreenInfo(orientation: Int) {
         var w: Int
         var h: Int
@@ -277,6 +288,12 @@ class MainService : Service() {
         Log.d(logTag,"updateScreenInfo:w:$w,h:$h")
         var scale = 1
         if (w != 0 && h != 0) {
+            if (isHardwareCodec == false && (w > MAX_SCREEN_SIZE || h > MAX_SCREEN_SIZE)) {
+                scale = 2
+                w /= scale
+                h /= scale
+                dpi /= scale
+            }
             if (SCREEN_INFO.width != w) {
                 SCREEN_INFO.width = w
                 SCREEN_INFO.height = h

libs/scrap/src/common/android.rs

@@ -7,7 +7,7 @@ use std::sync::Mutex;
 use std::{io, time::Duration};
 
 lazy_static! {
-    static ref SCREEN_SIZE: Mutex<(u16, u16, u16)> = Mutex::new((0, 0, 0)); // (width, height, scale)
+   pub(crate)  static ref SCREEN_SIZE: Mutex<(u16, u16, u16)> = Mutex::new((0, 0, 0)); // (width, height, scale)
 }
 
 pub struct Capturer {

libs/scrap/src/common/aom.rs

@@ -304,6 +304,10 @@ impl EncoderApi for AomEncoder {
     fn latency_free(&self) -> bool {
         true
     }
+
+    fn is_hardware(&self) -> bool {
+        false
+    }
 }
 
 impl AomEncoder {

libs/scrap/src/common/codec.rs

@@ -74,6 +74,8 @@ pub trait EncoderApi {
     fn support_changing_quality(&self) -> bool;
 
     fn latency_free(&self) -> bool;
+
+    fn is_hardware(&self) -> bool;
 }
 
 pub struct Encoder {

libs/scrap/src/common/convert.rs

@@ -213,6 +213,14 @@ pub fn convert_to_yuv(
         );
     }
     if src_pixfmt == crate::Pixfmt::BGRA || src_pixfmt == crate::Pixfmt::RGBA {
+        // stride is calculated, not real, so we need to check it
+        if src_stride[0] < src_width * 4 {
+            bail!(
+                "src_stride[0] < src_width * 4: {} < {}",
+                src_stride[0],
+                src_width * 4
+            );
+        }
         if src.len() < src_stride[0] * src_height {
             bail!(
                 "wrong src len, {} < {} * {}",

libs/scrap/src/common/hwcodec.rs

@@ -198,6 +198,10 @@ impl EncoderApi for HwRamEncoder {
     fn latency_free(&self) -> bool {
         !self.config.name.contains("mediacodec")
     }
+
+    fn is_hardware(&self) -> bool {
+        true
+    }
 }
 
 impl HwRamEncoder {

libs/scrap/src/common/mod.rs

@@ -481,3 +481,8 @@ pub trait GoogleImage {
         }
     }
 }
+
+#[cfg(target_os = "android")]
+pub fn screen_size() -> (u16, u16, u16) {
+    SCREEN_SIZE.lock().unwrap().clone()
+}

libs/scrap/src/common/vpxcodec.rs

@@ -242,6 +242,10 @@ impl EncoderApi for VpxEncoder {
     fn latency_free(&self) -> bool {
         true
     }
+
+    fn is_hardware(&self) -> bool {
+        false
+    }
 }
 
 impl VpxEncoder {

libs/scrap/src/common/vram.rs

@@ -188,6 +188,10 @@ impl EncoderApi for VRamEncoder {
     fn latency_free(&self) -> bool {
         true
     }
+
+    fn is_hardware(&self) -> bool {
+        true
+    }
 }
 
 impl VRamEncoder {

src/server/video_service.rs

@@ -444,6 +444,8 @@ fn run(vs: VideoService) -> ResultType<()> {
     };
     #[cfg(feature = "vram")]
     c.set_output_texture(encoder.input_texture());
+    #[cfg(target_os = "android")]
+    check_change_scale(encoder.is_hardware())?;
     VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate());
     VIDEO_QOS
         .lock()
@@ -493,22 +495,31 @@ fn run(vs: VideoService) -> ResultType<()> {
             }
         }
         if client_record != video_qos.record() {
+            log::info!("switch due to record changed");
             bail!("SWITCH");
         }
         drop(video_qos);
 
         if sp.is_option_true(OPTION_REFRESH) {
             let _ = try_broadcast_display_changed(&sp, display_idx, &c);
+            log::info!("switch to refresh");
             bail!("SWITCH");
         }
         if codec_format != Encoder::negotiated_codec() {
+            log::info!(
+                "switch due to codec changed, {:?} -> {:?}",
+                codec_format,
+                Encoder::negotiated_codec()
+            );
             bail!("SWITCH");
         }
         #[cfg(windows)]
         if last_portable_service_running != crate::portable_service::client::running() {
+            log::info!("switch due to portable service running changed");
             bail!("SWITCH");
         }
         if Encoder::use_i444(&encoder_cfg) != use_i444 {
+            log::info!("switch due to i444 changed");
             bail!("SWITCH");
         }
         #[cfg(all(windows, feature = "vram"))]
@@ -811,6 +822,26 @@ fn get_recorder(
     recorder
 }
 
+#[cfg(target_os = "android")]
+fn check_change_scale(hardware: bool) -> ResultType<()> {
+    let screen_size = scrap::screen_size();
+    log::info!("hardware: {hardware}, screen_size: {screen_size:?}",);
+    scrap::android::call_main_service_set_by_name(
+        "is_hardware_codec",
+        Some(hardware.to_string().as_str()),
+        None,
+    )
+    .ok();
+    let old_scale = screen_size.2;
+    let new_scale = scrap::screen_size().2;
+    if old_scale != new_scale {
+        log::info!("switch due to scale changed, {old_scale} -> {new_scale}");
+        // switch is not a must, but it is better to do so.
+        bail!("SWITCH");
+    }
+    Ok(())
+}
+
 fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> ResultType<()> {
     let privacy_mode_id_2 = get_privacy_mode_conn_id().unwrap_or(INVALID_PRIVACY_MODE_CONN_ID);
     if privacy_mode_id != privacy_mode_id_2 {
@@ -821,6 +852,7 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu
             );
             sp.send_to_others(msg_out, privacy_mode_id_2);
         }
+        log::info!("switch due to privacy mode changed");
         bail!("SWITCH");
     }
     Ok(())
@@ -838,6 +870,7 @@ fn handle_one_frame(
     sp.snapshot(|sps| {
         // so that new sub and old sub share the same encoder after switch
         if sps.has_subscribes() {
+            log::info!("switch due to new subscriber");
             bail!("SWITCH");
         }
         Ok(())
@@ -858,6 +891,7 @@ fn handle_one_frame(
         }
         Err(e) => match e.to_string().as_str() {
             scrap::codec::ENCODE_NEED_SWITCH => {
+                log::info!("switch due to encoder need switch");
                 bail!("SWITCH");
             }
             _ => {}