disable hardware encoding if encoding fails too many times (#8327)

Signed-off-by: 21pages <sunboeasy@gmail.com>

libs/scrap/src/common/aom.rs

@@ -308,6 +308,8 @@ impl EncoderApi for AomEncoder {
     fn is_hardware(&self) -> bool {
         false
     }
+
+    fn disable(&self) {}
 }
 
 impl AomEncoder {

libs/scrap/src/common/codec.rs

@@ -76,6 +76,8 @@ pub trait EncoderApi {
     fn latency_free(&self) -> bool;
 
     fn is_hardware(&self) -> bool;
+
+    fn disable(&self);
 }
 
 pub struct Encoder {
@@ -145,7 +147,6 @@ impl Encoder {
                 Err(e) => {
                     log::error!("new hw encoder failed: {e:?}, clear config");
                     HwCodecConfig::clear(false, true);
-                    Self::update(EncodingUpdate::Check);
                     *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
                     Err(e)
                 }
@@ -158,7 +159,6 @@ impl Encoder {
                 Err(e) => {
                     log::error!("new vram encoder failed: {e:?}, clear config");
                     HwCodecConfig::clear(true, true);
-                    Self::update(EncodingUpdate::Check);
                     *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
                     Err(e)
                 }

libs/scrap/src/common/hwcodec.rs

@@ -15,7 +15,7 @@ use hbb_common::{
 };
 use hwcodec::{
     common::{
-        get_gpu_signature, DataFormat,
+        DataFormat,
         Quality::{self, *},
         RateControl::{self, *},
     },
@@ -210,6 +210,10 @@ impl EncoderApi for HwRamEncoder {
     fn is_hardware(&self) -> bool {
         true
     }
+
+    fn disable(&self) {
+        HwCodecConfig::clear(false, true);
+    }
 }
 
 impl HwRamEncoder {
@@ -582,7 +586,7 @@ impl HwCodecConfig {
                 None => {
                     log::info!("try load cached hwcodec config");
                     let c = hbb_common::config::common_load::<HwCodecConfig>("_hwcodec");
-                    let new_signature = get_gpu_signature();
+                    let new_signature = hwcodec::common::get_gpu_signature();
                     if c.signature == new_signature {
                         log::debug!("load cached hwcodec config: {c:?}");
                         *CONFIG.lock().unwrap() = Some(c.clone());
@@ -647,6 +651,7 @@ impl HwCodecConfig {
                 }
             }
         }
+        crate::codec::Encoder::update(crate::codec::EncodingUpdate::Check);
     }
 }
 
@@ -679,7 +684,7 @@ pub fn check_available_hwcodec() -> String {
         vram_encode: vram.0,
         #[cfg(feature = "vram")]
         vram_decode: vram.1,
-        signature: get_gpu_signature(),
+        signature: hwcodec::common::get_gpu_signature(),
     };
     log::debug!("{c:?}");
     serde_json::to_string(&c).unwrap_or_default()

libs/scrap/src/common/vpxcodec.rs

@@ -246,6 +246,8 @@ impl EncoderApi for VpxEncoder {
     fn is_hardware(&self) -> bool {
         false
     }
+
+    fn disable(&self) {}
 }
 
 impl VpxEncoder {

libs/scrap/src/common/vram.rs

@@ -194,6 +194,10 @@ impl EncoderApi for VRamEncoder {
     fn is_hardware(&self) -> bool {
         true
     }
+
+    fn disable(&self) {
+        HwCodecConfig::clear(true, true);
+    }
 }
 
 impl VRamEncoder {

src/server/video_service.rs

@@ -481,6 +481,7 @@ fn run(vs: VideoService) -> ResultType<()> {
     let mut mid_data = Vec::new();
     let mut repeat_encode_counter = 0;
     let repeat_encode_max = 10;
+    let mut encode_fail_counter = 0;
 
     while sp.ok() {
         #[cfg(windows)]
@@ -568,6 +569,7 @@ fn run(vs: VideoService) -> ResultType<()> {
                         ms,
                         &mut encoder,
                         recorder.clone(),
+                        &mut encode_fail_counter,
                     )?;
                     frame_controller.set_send(now, send_conn_ids);
                 }
@@ -622,6 +624,7 @@ fn run(vs: VideoService) -> ResultType<()> {
                             ms,
                             &mut encoder,
                             recorder.clone(),
+                            &mut encode_fail_counter,
                         )?;
                         frame_controller.set_send(now, send_conn_ids);
                     }
@@ -876,6 +879,7 @@ fn handle_one_frame(
     ms: i64,
     encoder: &mut Encoder,
     recorder: Arc<Mutex<Option<Recorder>>>,
+    encode_fail_counter: &mut usize,
 ) -> ResultType<HashSet<i32>> {
     sp.snapshot(|sps| {
         // so that new sub and old sub share the same encoder after switch
@@ -889,6 +893,7 @@ fn handle_one_frame(
     let mut send_conn_ids: HashSet<i32> = Default::default();
     match encoder.encode_to_message(frame, ms) {
         Ok(mut vf) => {
+            *encode_fail_counter = 0;
             vf.display = display as _;
             let mut msg = Message::new();
             msg.set_video_frame(vf);
@@ -899,13 +904,29 @@ fn handle_one_frame(
                 .map(|r| r.write_message(&msg));
             send_conn_ids = sp.send_video_frame(msg);
         }
-        Err(e) => match e.to_string().as_str() {
+        Err(e) => {
-            scrap::codec::ENCODE_NEED_SWITCH => {
+            let max_fail_times = if cfg!(target_os = "android") && encoder.is_hardware() {
-                log::info!("switch due to encoder need switch");
+                12
-                bail!("SWITCH");
+            } else {
+                6
+            };
+            *encode_fail_counter += 1;
+            if *encode_fail_counter >= max_fail_times {
+                *encode_fail_counter = 0;
+                if encoder.is_hardware() {
+                    encoder.disable();
+                    log::error!("switch due to encoding fails more than {max_fail_times} times");
+                    bail!("SWITCH");
+                }
             }
-            _ => {}
+            match e.to_string().as_str() {
-        },
+                scrap::codec::ENCODE_NEED_SWITCH => {
+                    log::error!("switch due to encoder need switch");
+                    bail!("SWITCH");
+                }
+                _ => {}
+            }
+        }
     }
     Ok(send_conn_ids)
 }