Use fallback codec if first frame fails (#9242)

* Both encoding and decoding use fallback if first frame fails
* More aggresive max fail counter
* Update hwcodec, add judgement when length of the encoded data is zero, https://github.com/rustdesk/rustdesk-server-pro/discussions/382#discussioncomment-10525997
* Fix serde hwcodec config toml fails when the non-first vec![] is empty, https://github.com/toml-rs/toml-rs/issues/384, the config file is used for cache, when check process is not finished, the cache is used.
* Allow cm not start for pro user

Signed-off-by: 21pages <sunboeasy@gmail.com>

Cargo.lock

@@ -3045,7 +3045,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
 [[package]]
 name = "hwcodec"
 version = "0.7.0"
-source = "git+https://github.com/rustdesk-org/hwcodec#6abd1898f3a03481ed0c038507b5218d6ea94267"
+source = "git+https://github.com/rustdesk-org/hwcodec#b78a69c81631dd9ccfed9df68709808193082242"
 dependencies = [
  "bindgen 0.59.2",
  "cc",

libs/scrap/src/common/hwcodec.rs

@@ -498,6 +498,15 @@ pub struct HwCodecConfig {
     pub vram_decode: Vec<hwcodec::vram::DecodeContext>,
 }
 
+// HwCodecConfig2 is used to store the config in json format,
+// confy can't serde HwCodecConfig successfully if the non-first struct Vec is empty due to old toml version.
+// struct T { a: Vec<A>, b: Vec<String>} will fail if b is empty, but struct T { a: Vec<String>, b: Vec<String>} is ok.
+#[derive(Debug, Default, Serialize, Deserialize, Clone)]
+struct HwCodecConfig2 {
+    #[serde(default)]
+    pub config: String,
+}
+
 // ipc server process start check process once, other process get from ipc server once
 // install: --server start check process, check process send to --server,  ui get from --server
 // portable: ui start check process, check process send to ui
@@ -509,7 +518,12 @@ impl HwCodecConfig {
         log::info!("set hwcodec config");
         log::debug!("{config:?}");
         #[cfg(any(windows, target_os = "macos"))]
-        hbb_common::config::common_store(&config, "_hwcodec");
+        hbb_common::config::common_store(
+            &HwCodecConfig2 {
+                config: serde_json::to_string_pretty(&config).unwrap_or_default(),
+            },
+            "_hwcodec",
+        );
         *CONFIG.lock().unwrap() = Some(config);
         *CONFIG_SET_BY_IPC.lock().unwrap() = true;
     }
@@ -587,7 +601,8 @@ impl HwCodecConfig {
                 Some(c) => c,
                 None => {
                     log::info!("try load cached hwcodec config");
-                    let c = hbb_common::config::common_load::<HwCodecConfig>("_hwcodec");
+                    let c = hbb_common::config::common_load::<HwCodecConfig2>("_hwcodec");
+                    let c: HwCodecConfig = serde_json::from_str(&c.config).unwrap_or_default();
                     let new_signature = hwcodec::common::get_gpu_signature();
                     if c.signature == new_signature {
                         log::debug!("load cached hwcodec config: {c:?}");

src/client.rs

@@ -84,7 +84,7 @@ pub mod io_loop;
 pub const MILLI1: Duration = Duration::from_millis(1);
 pub const SEC30: Duration = Duration::from_secs(30);
 pub const VIDEO_QUEUE_SIZE: usize = 120;
-const MAX_DECODE_FAIL_COUNTER: usize = 10; // Currently, failed decode cause refresh_video, so make it small
+const MAX_DECODE_FAIL_COUNTER: usize = 3;
 
 #[cfg(target_os = "linux")]
 pub const LOGIN_MSG_DESKTOP_NOT_INITED: &str = "Desktop env is not inited";
@@ -1151,6 +1151,7 @@ pub struct VideoHandler {
     record: bool,
     _display: usize, // useful for debug
     fail_counter: usize,
+    first_frame: bool,
 }
 
 impl VideoHandler {
@@ -1176,6 +1177,7 @@ impl VideoHandler {
             record: false,
             _display,
             fail_counter: 0,
+            first_frame: true,
         }
     }
 
@@ -1204,9 +1206,19 @@ impl VideoHandler {
                     self.fail_counter = 0;
                 } else {
                     if self.fail_counter < usize::MAX {
-                        self.fail_counter += 1
+                        if self.first_frame && self.fail_counter < MAX_DECODE_FAIL_COUNTER {
+                            log::error!("decode first frame failed");
+                            self.fail_counter = MAX_DECODE_FAIL_COUNTER;
+                        } else {
+                            self.fail_counter += 1;
+                        }
+                        log::error!(
+                            "Failed to handle video frame, fail counter: {}",
+                            self.fail_counter
+                        );
                     }
                 }
+                self.first_frame = false;
                 if self.record {
                     self.recorder
                         .lock()
@@ -1222,12 +1234,17 @@ impl VideoHandler {
 
     /// Reset the decoder, change format if it is Some
     pub fn reset(&mut self, format: Option<CodecFormat>) {
+        log::info!(
+            "reset video handler for display #{}, format: {format:?}",
+            self._display
+        );
         #[cfg(target_os = "macos")]
         self.rgb.set_align(crate::get_dst_align_rgba());
         let luid = Self::get_adapter_luid();
         let format = format.unwrap_or(self.decoder.format());
         self.decoder = Decoder::new(format, luid);
         self.fail_counter = 0;
+        self.first_frame = true;
     }
 
     /// Start or stop screen record.

src/server/connection.rs

@@ -1670,9 +1670,11 @@ impl Connection {
                 .await
                 {
                     log::error!("ipc to connection manager exit: {}", err);
+                    // https://github.com/rustdesk/rustdesk-server-pro/discussions/382#discussioncomment-10525725, cm may start failed
                     #[cfg(windows)]
                     if !crate::platform::is_prelogin()
                         && !err.to_string().contains(crate::platform::EXPLORER_EXE)
+                        && !crate::hbbs_http::sync::is_pro()
                     {
                         allow_err!(tx_from_cm_clone.send(Data::CmErr(err.to_string())));
                     }

src/server/video_service.rs

@@ -486,6 +486,7 @@ fn run(vs: VideoService) -> ResultType<()> {
     let mut repeat_encode_counter = 0;
     let repeat_encode_max = 10;
     let mut encode_fail_counter = 0;
+    let mut first_frame = true;
 
     while sp.ok() {
         #[cfg(windows)]
@@ -574,6 +575,7 @@ fn run(vs: VideoService) -> ResultType<()> {
                         &mut encoder,
                         recorder.clone(),
                         &mut encode_fail_counter,
+                        &mut first_frame,
                     )?;
                     frame_controller.set_send(now, send_conn_ids);
                 }
@@ -629,6 +631,7 @@ fn run(vs: VideoService) -> ResultType<()> {
                             &mut encoder,
                             recorder.clone(),
                             &mut encode_fail_counter,
+                            &mut first_frame,
                         )?;
                         frame_controller.set_send(now, send_conn_ids);
                     }
@@ -906,6 +909,7 @@ fn handle_one_frame(
     encoder: &mut Encoder,
     recorder: Arc<Mutex<Option<Recorder>>>,
     encode_fail_counter: &mut usize,
+    first_frame: &mut bool,
 ) -> ResultType<HashSet<i32>> {
     sp.snapshot(|sps| {
         // so that new sub and old sub share the same encoder after switch
@@ -917,6 +921,8 @@ fn handle_one_frame(
     })?;
 
     let mut send_conn_ids: HashSet<i32> = Default::default();
+    let first = *first_frame;
+    *first_frame = false;
     match encoder.encode_to_message(frame, ms) {
         Ok(mut vf) => {
             *encode_fail_counter = 0;
@@ -931,17 +937,21 @@ fn handle_one_frame(
             send_conn_ids = sp.send_video_frame(msg);
         }
         Err(e) => {
-            let max_fail_times = if cfg!(target_os = "android") && encoder.is_hardware() {
-                12
-            } else {
-                6
-            };
             *encode_fail_counter += 1;
-            if *encode_fail_counter >= max_fail_times {
+            // Encoding errors are not frequent except on Android
+            if !cfg!(target_os = "android") {
+                log::error!("encode fail: {e:?}, times: {}", *encode_fail_counter,);
+            }
+            let max_fail_times = if cfg!(target_os = "android") && encoder.is_hardware() {
+                9
+            } else {
+                3
+            };
+            if first || *encode_fail_counter >= max_fail_times {
                 *encode_fail_counter = 0;
                 if encoder.is_hardware() {
                     encoder.disable();
-                    log::error!("switch due to encoding fails more than {max_fail_times} times");
+                    log::error!("switch due to encoding fails, first frame: {first}, error: {e:?}");
                     bail!("SWITCH");
                 }
             }