video handler holds only one decoder of the current codec format (#6939)

1. For example: when receiving h264 video frames, only 1 decoder is created, vram > ram
2. For creation and decoding failed:
  * Remove real_supported_decodings, this will update real existing decoders, replace it with the "mark_unsupported" vector. After creating the decoder failure, marks the codec as unsupported and updates supported decoding to the controlled side
  *  Add `fail_counter` in the decoder. When decoding 10 consecutive frames failed, adding codec type to 'mark_unsupported' vector
  *  The controlled end always ignores the unavailability of VP9

Signed-off-by: 21pages <pages21@163.com>

libs/scrap/examples/benchmark.rs

@@ -240,7 +240,10 @@ fn test_av1(
 #[cfg(feature = "hwcodec")]
 mod hw {
     use hwcodec::ffmpeg::CodecInfo;
-    use scrap::hwcodec::{HwDecoder, HwEncoder, HwEncoderConfig};
+    use scrap::{
+        hwcodec::{HwDecoder, HwEncoder, HwEncoderConfig},
+        CodecFormat,
+    };
 
     use super::*;
 
@@ -254,13 +257,8 @@ mod hw {
         if let Some(info) = best.h265 {
             test_encoder(width, height, quality, info, c, yuv_count, &mut h265s);
         }
-        let best = HwDecoder::best();
+        test_decoder(CodecFormat::H264, &h264s);
-        if let Some(info) = best.h264 {
+        test_decoder(CodecFormat::H265, &h265s);
-            test_decoder(info, &h264s);
-        }
-        if let Some(info) = best.h265 {
-            test_decoder(info, &h265s);
-        }
     }
 
     fn test_encoder(
@@ -322,16 +320,21 @@ mod hw {
         );
     }
 
-    fn test_decoder(info: CodecInfo, h26xs: &Vec<Vec<u8>>) {
+    fn test_decoder(format: CodecFormat, h26xs: &Vec<Vec<u8>>) {
-        let mut decoder = HwDecoder::new(info.clone()).unwrap();
+        let mut decoder = HwDecoder::new(format).unwrap();
         let start = Instant::now();
         let mut cnt = 0;
         for h26x in h26xs {
             let _ = decoder.decode(h26x).unwrap();
             cnt += 1;
         }
-        let device = format!("{:?}", info.hwdevice).to_lowercase();
+        let device = format!("{:?}", decoder.info.hwdevice).to_lowercase();
         let device = device.split("_").last().unwrap();
-        println!("{} {}: {:?}", info.name, device, start.elapsed() / cnt);
+        println!(
+            "{} {}: {:?}",
+            decoder.info.name,
+            device,
+            start.elapsed() / cnt
+        );
     }
 }

libs/scrap/src/common/codec.rs

@@ -10,14 +10,12 @@ use crate::gpucodec::*;
 #[cfg(feature = "hwcodec")]
 use crate::hwcodec::*;
 #[cfg(feature = "mediacodec")]
-use crate::mediacodec::{
+use crate::mediacodec::{MediaCodecDecoder, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT};
-    MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
-};
 use crate::{
     aom::{self, AomDecoder, AomEncoder, AomEncoderConfig},
     common::GoogleImage,
     vpxcodec::{self, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, VpxVideoCodecId},
-    CodecName, EncodeInput, EncodeYuvFormat, ImageRgb,
+    CodecFormat, CodecName, EncodeInput, EncodeYuvFormat, ImageRgb,
 };
 
 use hbb_common::{
@@ -96,13 +94,21 @@ pub struct Decoder {
     vp9: Option<VpxDecoder>,
     av1: Option<AomDecoder>,
     #[cfg(feature = "hwcodec")]
-    hw: HwDecoders,
+    h264_ram: Option<HwDecoder>,
+    #[cfg(feature = "hwcodec")]
+    h265_ram: Option<HwDecoder>,
     #[cfg(feature = "gpucodec")]
-    gpu: GpuDecoders,
+    h264_vram: Option<GpuDecoder>,
+    #[cfg(feature = "gpucodec")]
+    h265_vram: Option<GpuDecoder>,
+    #[cfg(feature = "mediacodec")]
+    h264_media_codec: MediaCodecDecoder,
+    #[cfg(feature = "mediacodec")]
+    h265_media_codec: MediaCodecDecoder,
+    format: CodecFormat,
+    valid: bool,
     #[cfg(feature = "hwcodec")]
     i420: Vec<u8>,
-    #[cfg(feature = "mediacodec")]
-    media_codec: MediaCodecDecoders,
 }
 
 #[derive(Debug, Clone)]
@@ -372,6 +378,7 @@ impl Decoder {
         id_for_perfer: Option<&str>,
         _flutter: bool,
         _luid: Option<i64>,
+        mark_unsupported: &Vec<CodecFormat>,
     ) -> SupportedDecoding {
         let (prefer, prefer_chroma) = Self::preference(id_for_perfer);
 
@@ -398,12 +405,12 @@ impl Decoder {
         }
         #[cfg(feature = "gpucodec")]
         if enable_gpucodec_option() && _flutter {
-            decoding.ability_h264 |= if GpuDecoder::available(CodecName::H264GPU, _luid).len() > 0 {
+            decoding.ability_h264 |= if GpuDecoder::available(CodecFormat::H264, _luid).len() > 0 {
                 1
             } else {
                 0
             };
-            decoding.ability_h265 |= if GpuDecoder::available(CodecName::H265GPU, _luid).len() > 0 {
+            decoding.ability_h265 |= if GpuDecoder::available(CodecFormat::H265, _luid).len() > 0 {
                 1
             } else {
                 0
@@ -424,72 +431,148 @@ impl Decoder {
                     0
                 };
         }
+        for unsupported in mark_unsupported {
+            match unsupported {
+                CodecFormat::VP8 => decoding.ability_vp8 = 0,
+                CodecFormat::VP9 => decoding.ability_vp9 = 0,
+                CodecFormat::AV1 => decoding.ability_av1 = 0,
+                CodecFormat::H264 => decoding.ability_h264 = 0,
+                CodecFormat::H265 => decoding.ability_h265 = 0,
+                _ => {}
+            }
+        }
         decoding
     }
 
-    pub fn exist_codecs(&self, _flutter: bool) -> CodecAbility {
+    pub fn new(format: CodecFormat, _luid: Option<i64>) -> Decoder {
-        #[allow(unused_mut)]
+        log::info!("try create new decoder, format: {format:?}, _luid: {_luid:?}");
-        let mut ability = CodecAbility {
+        let (mut vp8, mut vp9, mut av1) = (None, None, None);
-            vp8: self.vp8.is_some(),
-            vp9: self.vp9.is_some(),
-            av1: self.av1.is_some(),
-            ..Default::default()
-        };
         #[cfg(feature = "hwcodec")]
-        {
+        let (mut h264_ram, mut h265_ram) = (None, None);
-            ability.h264 |= self.hw.h264.is_some();
-            ability.h265 |= self.hw.h265.is_some();
-        }
         #[cfg(feature = "gpucodec")]
-        if _flutter {
+        let (mut h264_vram, mut h265_vram) = (None, None);
-            ability.h264 |= self.gpu.h264.is_some();
-            ability.h265 |= self.gpu.h265.is_some();
-        }
         #[cfg(feature = "mediacodec")]
-        {
+        let (mut h264_media_codec, mut h265_media_codec) = (None, None);
-            ability.h264 = self.media_codec.h264.is_some();
+        let mut valid = false;
-            ability.h265 = self.media_codec.h265.is_some();
-        }
-        ability
-    }
 
-    pub fn new(_luid: Option<i64>) -> Decoder {
+        match format {
-        let vp8 = VpxDecoder::new(VpxDecoderConfig {
+            CodecFormat::VP8 => {
-            codec: VpxVideoCodecId::VP8,
+                match VpxDecoder::new(VpxDecoderConfig {
-        })
+                    codec: VpxVideoCodecId::VP8,
-        .ok();
+                }) {
-        let vp9 = VpxDecoder::new(VpxDecoderConfig {
+                    Ok(v) => vp8 = Some(v),
-            codec: VpxVideoCodecId::VP9,
+                    Err(e) => log::error!("create VP8 decoder failed: {}", e),
-        })
+                }
-        .ok();
+                valid = vp8.is_some();
-        let av1 = AomDecoder::new().ok();
+            }
+            CodecFormat::VP9 => {
+                match VpxDecoder::new(VpxDecoderConfig {
+                    codec: VpxVideoCodecId::VP9,
+                }) {
+                    Ok(v) => vp9 = Some(v),
+                    Err(e) => log::error!("create VP9 decoder failed: {}", e),
+                }
+                valid = vp9.is_some();
+            }
+            CodecFormat::AV1 => {
+                match AomDecoder::new() {
+                    Ok(v) => av1 = Some(v),
+                    Err(e) => log::error!("create AV1 decoder failed: {}", e),
+                }
+                valid = av1.is_some();
+            }
+            CodecFormat::H264 => {
+                #[cfg(feature = "gpucodec")]
+                if !valid && enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
+                    match GpuDecoder::new(format, _luid) {
+                        Ok(v) => h264_vram = Some(v),
+                        Err(e) => log::error!("create H264 vram decoder failed: {}", e),
+                    }
+                    valid = h264_vram.is_some();
+                }
+                #[cfg(feature = "hwcodec")]
+                if !valid && enable_hwcodec_option() {
+                    match HwDecoder::new(format) {
+                        Ok(v) => h264_ram = Some(v),
+                        Err(e) => log::error!("create H264 ram decoder failed: {}", e),
+                    }
+                    valid = h264_ram.is_some();
+                }
+                #[cfg(feature = "mediacodec")]
+                if !valid && enable_hwcodec_option() {
+                    h264_media_codec = MediaCodecDecoder::new(format);
+                    if h264_media_codec.is_none() {
+                        log::error!("create H264 media codec decoder failed");
+                    }
+                    valid = h264_media_codec.is_some();
+                }
+            }
+            CodecFormat::H265 => {
+                #[cfg(feature = "gpucodec")]
+                if !valid && enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
+                    match GpuDecoder::new(format, _luid) {
+                        Ok(v) => h265_vram = Some(v),
+                        Err(e) => log::error!("create H265 vram decoder failed: {}", e),
+                    }
+                    valid = h265_vram.is_some();
+                }
+                #[cfg(feature = "hwcodec")]
+                if !valid && enable_hwcodec_option() {
+                    match HwDecoder::new(format) {
+                        Ok(v) => h265_ram = Some(v),
+                        Err(e) => log::error!("create H265 ram decoder failed: {}", e),
+                    }
+                    valid = h265_ram.is_some();
+                }
+                #[cfg(feature = "mediacodec")]
+                if !valid && enable_hwcodec_option() {
+                    h265_media_codec = MediaCodecDecoder::new(format);
+                    if h265_media_codec.is_none() {
+                        log::error!("create H265 media codec decoder failed");
+                    }
+                    valid = h265_media_codec.is_some();
+                }
+            }
+            CodecFormat::Unknown => {
+                log::error!("unknown codec format, cannot create decoder");
+            }
+        }
+        if !valid {
+            log::error!("failed to create {format:?} decoder");
+        } else {
+            log::info!("create {format:?} decoder success");
+        }
         Decoder {
             vp8,
             vp9,
             av1,
             #[cfg(feature = "hwcodec")]
-            hw: if enable_hwcodec_option() {
+            h264_ram,
-                HwDecoder::new_decoders()
+            #[cfg(feature = "hwcodec")]
-            } else {
+            h265_ram,
-                HwDecoders::default()
-            },
             #[cfg(feature = "gpucodec")]
-            gpu: if enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
+            h264_vram,
-                GpuDecoder::new_decoders(_luid)
+            #[cfg(feature = "gpucodec")]
-            } else {
+            h265_vram,
-                GpuDecoders::default()
+            #[cfg(feature = "mediacodec")]
-            },
+            h264_media_codec,
+            #[cfg(feature = "mediacodec")]
+            h265_media_codec,
+            format,
+            valid,
             #[cfg(feature = "hwcodec")]
             i420: vec![],
-            #[cfg(feature = "mediacodec")]
-            media_codec: if enable_hwcodec_option() {
-                MediaCodecDecoder::new_decoders()
-            } else {
-                MediaCodecDecoders::default()
-            },
         }
     }
 
+    pub fn format(&self) -> CodecFormat {
+        self.format
+    }
+
+    pub fn valid(&self) -> bool {
+        self.valid
+    }
+
     // rgb [in/out] fmt and stride must be set in ImageRgb
     pub fn handle_video_frame(
         &mut self,
@@ -525,12 +608,12 @@ impl Decoder {
             video_frame::Union::H264s(h264s) => {
                 *chroma = Some(Chroma::I420);
                 #[cfg(feature = "gpucodec")]
-                if let Some(decoder) = &mut self.gpu.h264 {
+                if let Some(decoder) = &mut self.h264_vram {
                     *_pixelbuffer = false;
                     return Decoder::handle_gpu_video_frame(decoder, h264s, _texture);
                 }
                 #[cfg(feature = "hwcodec")]
-                if let Some(decoder) = &mut self.hw.h264 {
+                if let Some(decoder) = &mut self.h264_ram {
                     return Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420);
                 }
                 Err(anyhow!("don't support h264!"))
@@ -539,12 +622,12 @@ impl Decoder {
             video_frame::Union::H265s(h265s) => {
                 *chroma = Some(Chroma::I420);
                 #[cfg(feature = "gpucodec")]
-                if let Some(decoder) = &mut self.gpu.h265 {
+                if let Some(decoder) = &mut self.h265_vram {
                     *_pixelbuffer = false;
                     return Decoder::handle_gpu_video_frame(decoder, h265s, _texture);
                 }
                 #[cfg(feature = "hwcodec")]
-                if let Some(decoder) = &mut self.hw.h265 {
+                if let Some(decoder) = &mut self.h265_ram {
                     return Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420);
                 }
                 Err(anyhow!("don't support h265!"))
@@ -552,7 +635,7 @@ impl Decoder {
             #[cfg(feature = "mediacodec")]
             video_frame::Union::H264s(h264s) => {
                 *chroma = Some(Chroma::I420);
-                if let Some(decoder) = &mut self.media_codec.h264 {
+                if let Some(decoder) = &mut self.h264_media_codec {
                     Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
                 } else {
                     Err(anyhow!("don't support h264!"))
@@ -561,7 +644,7 @@ impl Decoder {
             #[cfg(feature = "mediacodec")]
             video_frame::Union::H265s(h265s) => {
                 *chroma = Some(Chroma::I420);
-                if let Some(decoder) = &mut self.media_codec.h265 {
+                if let Some(decoder) = &mut self.h265_media_codec {
                     Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
                 } else {
                     Err(anyhow!("don't support h265!"))

libs/scrap/src/common/gpucodec.rs

@@ -6,7 +6,7 @@ use std::{
 
 use crate::{
     codec::{base_bitrate, enable_gpucodec_option, EncoderApi, EncoderCfg, Quality},
-    AdapterDevice, CodecName, EncodeInput, EncodeYuvFormat, Pixfmt,
+    AdapterDevice, CodecFormat, CodecName, EncodeInput, EncodeYuvFormat, Pixfmt,
 };
 use gpucodec::gpu_common::{
     self, Available, DecodeContext, DynamicContext, EncodeContext, FeatureContext, MAX_GOP,
@@ -87,7 +87,10 @@ impl EncoderApi for GpuEncoder {
                         last_frame_len: 0,
                         same_bad_len_counter: 0,
                     }),
-                    Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
+                    Err(_) => {
+                        hbb_common::config::GpucodecConfig::clear();
+                        Err(anyhow!(format!("Failed to create encoder")))
+                    }
                 }
             }
             _ => Err(anyhow!("encoder type mismatch")),
@@ -300,8 +303,8 @@ pub struct GpuDecoders {
 }
 
 impl GpuDecoder {
-    pub fn try_get(name: CodecName, luid: Option<i64>) -> Option<DecodeContext> {
+    pub fn try_get(format: CodecFormat, luid: Option<i64>) -> Option<DecodeContext> {
-        let v: Vec<_> = Self::available(name, luid);
+        let v: Vec<_> = Self::available(format, luid);
         if v.len() > 0 {
             Some(v[0].clone())
         } else {
@@ -309,11 +312,11 @@ impl GpuDecoder {
         }
     }
 
-    pub fn available(name: CodecName, luid: Option<i64>) -> Vec<DecodeContext> {
+    pub fn available(format: CodecFormat, luid: Option<i64>) -> Vec<DecodeContext> {
         let luid = luid.unwrap_or_default();
-        let data_format = match name {
+        let data_format = match format {
-            CodecName::H264GPU => gpu_common::DataFormat::H264,
+            CodecFormat::H264 => gpu_common::DataFormat::H264,
-            CodecName::H265GPU => gpu_common::DataFormat::H265,
+            CodecFormat::H265 => gpu_common::DataFormat::H265,
             _ => return vec![],
         };
         get_available_config()
@@ -337,28 +340,18 @@ impl GpuDecoder {
         )
     }
 
-    pub fn new_decoders(luid: Option<i64>) -> GpuDecoders {
+    pub fn new(format: CodecFormat, luid: Option<i64>) -> ResultType<Self> {
-        let mut h264: Option<GpuDecoder> = None;
+        log::info!("try create {format:?} vram decoder, luid: {luid:?}");
-        let mut h265: Option<GpuDecoder> = None;
+        let ctx = Self::try_get(format, luid).ok_or(anyhow!("Failed to get decode context"))?;
-        if let Ok(decoder) = GpuDecoder::new(CodecName::H264GPU, luid) {
-            h264 = Some(decoder);
-        }
-        if let Ok(decoder) = GpuDecoder::new(CodecName::H265GPU, luid) {
-            h265 = Some(decoder);
-        }
-        log::info!(
-            "new gpu decoders, support h264: {}, h265: {}",
-            h264.is_some(),
-            h265.is_some()
-        );
-        GpuDecoders { h264, h265 }
-    }
-
-    pub fn new(name: CodecName, luid: Option<i64>) -> ResultType<Self> {
-        let ctx = Self::try_get(name, luid).ok_or(anyhow!("Failed to get decode context"))?;
         match Decoder::new(ctx) {
             Ok(decoder) => Ok(Self { decoder }),
-            Err(_) => Err(anyhow!(format!("Failed to create decoder"))),
+            Err(_) => {
+                hbb_common::config::GpucodecConfig::clear();
+                Err(anyhow!(format!(
+                    "Failed to create decoder, format: {:?}",
+                    format
+                )))
+            }
         }
     }
     pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<GpuDecoderImage>> {

libs/scrap/src/common/hwcodec.rs

@@ -1,10 +1,10 @@
 use crate::{
     codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg, Quality as Q},
-    hw, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
+    hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
     allow_err,
-    anyhow::{anyhow, Context},
+    anyhow::{anyhow, bail, Context},
     bytes::Bytes,
     config::HwCodecConfig,
     log,
@@ -94,7 +94,10 @@ impl EncoderApi for HwEncoder {
                         height: ctx.height as _,
                         bitrate,
                     }),
-                    Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
+                    Err(_) => {
+                        HwCodecConfig::clear();
+                        Err(anyhow!(format!("Failed to create encoder")))
+                    }
                 }
             }
             _ => Err(anyhow!("encoder type mismatch")),
@@ -230,31 +233,26 @@ impl HwDecoder {
         })
     }
 
-    pub fn new_decoders() -> HwDecoders {
+    pub fn new(format: CodecFormat) -> ResultType<Self> {
+        log::info!("try create {format:?} ram decoder");
         let best = HwDecoder::best();
-        let mut h264: Option<HwDecoder> = None;
+        let info = match format {
-        let mut h265: Option<HwDecoder> = None;
+            CodecFormat::H264 => {
-        let mut fail = false;
+                if let Some(info) = best.h264 {
-
+                    info
-        if let Some(info) = best.h264 {
+                } else {
-            h264 = HwDecoder::new(info).ok();
+                    bail!("no h264 decoder, should not be here");
-            if h264.is_none() {
+                }
-                fail = true;
             }
-        }
+            CodecFormat::H265 => {
-        if let Some(info) = best.h265 {
+                if let Some(info) = best.h265 {
-            h265 = HwDecoder::new(info).ok();
+                    info
-            if h265.is_none() {
+                } else {
-                fail = true;
+                    bail!("no h265 decoder, should not be here");
+                }
             }
-        }
+            _ => bail!("unsupported format: {:?}", format),
-        if fail {
+        };
-            hwcodec_new_check_process();
-        }
-        HwDecoders { h264, h265 }
-    }
-
-    pub fn new(info: CodecInfo) -> ResultType<Self> {
         let ctx = DecodeContext {
             name: info.name.clone(),
             device_type: info.hwdevice.clone(),
@@ -262,7 +260,10 @@ impl HwDecoder {
         };
         match Decoder::new(ctx) {
             Ok(decoder) => Ok(HwDecoder { decoder, info }),
-            Err(_) => Err(anyhow!(format!("Failed to create decoder"))),
+            Err(_) => {
+                HwCodecConfig::clear();
+                Err(anyhow!(format!("Failed to create decoder")))
+            }
         }
     }
     pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<HwDecoderImage>> {

libs/scrap/src/common/mediacodec.rs

@@ -10,7 +10,7 @@ use std::{
 use crate::ImageFormat;
 use crate::{
     codec::{EncoderApi, EncoderCfg},
-    I420ToABGR, I420ToARGB, ImageRgb,
+    CodecFormat, I420ToABGR, I420ToARGB, ImageRgb,
 };
 
 /// MediaCodec mime type name
@@ -37,17 +37,16 @@ impl Deref for MediaCodecDecoder {
     }
 }
 
-#[derive(Default)]
-pub struct MediaCodecDecoders {
-    pub h264: Option<MediaCodecDecoder>,
-    pub h265: Option<MediaCodecDecoder>,
-}
-
 impl MediaCodecDecoder {
-    pub fn new_decoders() -> MediaCodecDecoders {
+    pub fn new(format: CodecFormat) -> Option<MediaCodecDecoder> {
-        let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder);
+        match format {
-        let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder);
+            CodecFormat::H264 => create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder),
-        MediaCodecDecoders { h264, h265 }
+            CodecFormat::H265 => create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder),
+            _ => {
+                log::error!("Unsupported codec format: {}", format);
+                None
+            }
+        }
     }
 
     // rgb [in/out] fmt and stride must be set in ImageRgb

libs/scrap/src/common/mod.rs

@@ -251,7 +251,7 @@ pub enum CodecName {
     H265GPU,
 }
 
-#[derive(PartialEq, Debug, Clone)]
+#[derive(PartialEq, Debug, Clone, Copy)]
 pub enum CodecFormat {
     VP8,
     VP9,

src/client.rs

@@ -51,7 +51,7 @@ pub use helper::*;
 use scrap::{
     codec::Decoder,
     record::{Recorder, RecorderContext},
-    ImageFormat, ImageRgb,
+    CodecFormat, ImageFormat, ImageRgb,
 };
 
 use crate::{
@@ -76,6 +76,7 @@ pub mod io_loop;
 pub const MILLI1: Duration = Duration::from_millis(1);
 pub const SEC30: Duration = Duration::from_secs(30);
 pub const VIDEO_QUEUE_SIZE: usize = 120;
+const MAX_DECODE_FAIL_COUNTER: usize = 10; // Currently, failed decode cause refresh_video, so make it small
 
 #[cfg(all(target_os = "linux", feature = "linux_headless"))]
 #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -1027,24 +1028,25 @@ pub struct VideoHandler {
     recorder: Arc<Mutex<Option<Recorder>>>,
     record: bool,
     _display: usize, // useful for debug
+    fail_counter: usize,
 }
 
 impl VideoHandler {
     /// Create a new video handler.
-    pub fn new(_display: usize) -> Self {
+    pub fn new(format: CodecFormat, _display: usize) -> Self {
         #[cfg(all(feature = "gpucodec", feature = "flutter"))]
         let luid = crate::flutter::get_adapter_luid();
         #[cfg(not(all(feature = "gpucodec", feature = "flutter")))]
         let luid = Default::default();
-        println!("new session_get_adapter_luid: {:?}", luid);
+        log::info!("new video handler for display #{_display}, format: {format:?}, luid: {luid:?}");
-        log::info!("new video handler for display #{_display}");
         VideoHandler {
-            decoder: Decoder::new(luid),
+            decoder: Decoder::new(format, luid),
             rgb: ImageRgb::new(ImageFormat::ARGB, crate::DST_STRIDE_RGBA),
             texture: std::ptr::null_mut(),
             recorder: Default::default(),
             record: false,
             _display,
+            fail_counter: 0,
         }
     }
 
@@ -1056,6 +1058,10 @@ impl VideoHandler {
         pixelbuffer: &mut bool,
         chroma: &mut Option<Chroma>,
     ) -> ResultType<bool> {
+        let format = CodecFormat::from(&vf);
+        if format != self.decoder.format() {
+            self.reset(Some(format));
+        }
         match &vf.union {
             Some(frame) => {
                 let res = self.decoder.handle_video_frame(
@@ -1065,6 +1071,13 @@ impl VideoHandler {
                     pixelbuffer,
                     chroma,
                 );
+                if res.as_ref().is_ok_and(|x| *x) {
+                    self.fail_counter = 0;
+                } else {
+                    if self.fail_counter < usize::MAX {
+                        self.fail_counter += 1
+                    }
+                }
                 if self.record {
                     self.recorder
                         .lock()
@@ -1078,13 +1091,15 @@ impl VideoHandler {
         }
     }
 
-    /// Reset the decoder.
+    /// Reset the decoder, change format if it is Some
-    pub fn reset(&mut self) {
+    pub fn reset(&mut self, format: Option<CodecFormat>) {
         #[cfg(all(feature = "flutter", feature = "gpucodec"))]
         let luid = crate::flutter::get_adapter_luid();
         #[cfg(not(all(feature = "flutter", feature = "gpucodec")))]
         let luid = None;
-        self.decoder = Decoder::new(luid);
+        let format = format.unwrap_or(self.decoder.format());
+        self.decoder = Decoder::new(format, luid);
+        self.fail_counter = 0;
     }
 
     /// Start or stop screen record.
@@ -1133,6 +1148,7 @@ pub struct LoginConfigHandler {
     pub other_server: Option<(String, String, String)>,
     pub custom_fps: Arc<Mutex<Option<usize>>>,
     pub adapter_luid: Option<i64>,
+    pub mark_unsupported: Vec<CodecFormat>,
 }
 
 impl Deref for LoginConfigHandler {
@@ -1562,6 +1578,7 @@ impl LoginConfigHandler {
                 Some(&self.id),
                 cfg!(feature = "flutter"),
                 self.adapter_luid,
+                &self.mark_unsupported,
             ));
         n += 1;
 
@@ -1947,6 +1964,7 @@ impl LoginConfigHandler {
             Some(&self.id),
             cfg!(feature = "flutter"),
             self.adapter_luid,
+            &self.mark_unsupported,
         );
         let mut misc = Misc::new();
         misc.set_option(OptionMessage {
@@ -1958,44 +1976,6 @@ impl LoginConfigHandler {
         msg_out
     }
 
-    fn real_supported_decodings(
-        &self,
-        handler_controller_map: &Vec<VideoHandlerController>,
-    ) -> Data {
-        let abilities: Vec<CodecAbility> = handler_controller_map
-            .iter()
-            .map(|h| h.handler.decoder.exist_codecs(cfg!(feature = "flutter")))
-            .collect();
-        let all = |ability: fn(&CodecAbility) -> bool| -> i32 {
-            if abilities.iter().all(|d| ability(d)) {
-                1
-            } else {
-                0
-            }
-        };
-        let decoding = scrap::codec::Decoder::supported_decodings(
-            Some(&self.id),
-            cfg!(feature = "flutter"),
-            self.adapter_luid,
-        );
-        let decoding = SupportedDecoding {
-            ability_vp8: all(|e| e.vp8),
-            ability_vp9: all(|e| e.vp9),
-            ability_av1: all(|e| e.av1),
-            ability_h264: all(|e| e.h264),
-            ability_h265: all(|e| e.h265),
-            ..decoding
-        };
-        let mut misc = Misc::new();
-        misc.set_option(OptionMessage {
-            supported_decoding: hbb_common::protobuf::MessageField::some(decoding),
-            ..Default::default()
-        });
-        let mut msg_out = Message::new();
-        msg_out.set_misc(misc);
-        Data::Message(msg_out)
-    }
-
     pub fn restart_remote_device(&self) -> Message {
         let mut misc = Misc::new();
         misc.set_restart_remote_device(true);
@@ -2088,27 +2068,17 @@ where
                         };
                         let display = vf.display as usize;
                         let start = std::time::Instant::now();
-                        let mut created_new_handler = false;
+                        let format = CodecFormat::from(&vf);
                         if handler_controller_map.len() <= display {
                             for _i in handler_controller_map.len()..=display {
                                 handler_controller_map.push(VideoHandlerController {
-                                    handler: VideoHandler::new(_i),
+                                    handler: VideoHandler::new(format, _i),
                                     count: 0,
                                     duration: std::time::Duration::ZERO,
                                     skip_beginning: 0,
                                 });
-                                created_new_handler = true;
                             }
                         }
-                        if created_new_handler {
-                            session.send(
-                                session
-                                    .lc
-                                    .read()
-                                    .unwrap()
-                                    .real_supported_decodings(&handler_controller_map),
-                            );
-                        }
                         if let Some(handler_controller) = handler_controller_map.get_mut(display) {
                             let mut pixelbuffer = true;
                             let mut tmp_chroma = None;
@@ -2172,17 +2142,32 @@ where
                                 _ => {}
                             }
                         }
+
+                        // check invalid decoders
+                        let mut should_update_supported = false;
+                        handler_controller_map
+                            .iter()
+                            .map(|h| {
+                                if !h.handler.decoder.valid() || h.handler.fail_counter >= MAX_DECODE_FAIL_COUNTER {
+                                    let mut lc = session.lc.write().unwrap();
+                                    let format = h.handler.decoder.format();
+                                    if !lc.mark_unsupported.contains(&format) {
+                                        lc.mark_unsupported.push(format);
+                                        should_update_supported = true;
+                                        log::info!("mark {format:?} decoder as unsupported, valid:{}, fail_counter:{}, all unsupported:{:?}", h.handler.decoder.valid(), h.handler.fail_counter, lc.mark_unsupported);
+                                    }
+                                }
+                            })
+                            .count();
+                        if should_update_supported {
+                            session.send(Data::Message(
+                                session.lc.read().unwrap().update_supported_decodings(),
+                            ));
+                        }
                     }
                     MediaData::Reset(display) => {
                         if let Some(handler_controler) = handler_controller_map.get_mut(display) {
-                            handler_controler.handler.reset();
+                            handler_controler.handler.reset(None);
-                            session.send(
-                                session
-                                    .lc
-                                    .read()
-                                    .unwrap()
-                                    .real_supported_decodings(&handler_controller_map),
-                            );
                         }
                     }
                     MediaData::RecordScreen(start, display, w, h, id) => {

src/ui_interface.rs

@@ -846,7 +846,7 @@ pub fn has_gpucodec() -> bool {
 #[cfg(feature = "flutter")]
 #[inline]
 pub fn supported_hwdecodings() -> (bool, bool) {
-    let decoding = scrap::codec::Decoder::supported_decodings(None, true, None);
+    let decoding = scrap::codec::Decoder::supported_decodings(None, true, None, &vec![]);
     #[allow(unused_mut)]
     let (mut h264, mut h265) = (decoding.ability_h264 > 0, decoding.ability_h265 > 0);
     #[cfg(feature = "gpucodec")]

src/ui_session_interface.rs

@@ -437,8 +437,13 @@ impl<T: InvokeUiSession> Session<T> {
 
     pub fn alternative_codecs(&self) -> (bool, bool, bool, bool) {
         let luid = self.lc.read().unwrap().adapter_luid;
-        let decoder =
+        let mark_unsupported = self.lc.read().unwrap().mark_unsupported.clone();
-            scrap::codec::Decoder::supported_decodings(None, cfg!(feature = "flutter"), luid);
+        let decoder = scrap::codec::Decoder::supported_decodings(
+            None,
+            cfg!(feature = "flutter"),
+            luid,
+            &mark_unsupported,
+        );
         let mut vp8 = decoder.ability_vp8 > 0;
         let mut av1 = decoder.ability_av1 > 0;
         let mut h264 = decoder.ability_h264 > 0;