Show current codec in menu when auto codec is chosen (#7942)

* change negotiated codec name to negotiated codec format

Signed-off-by: 21pages <pages21@163.com>

* fallback to vp9 directly if failed to create encoder

Current fallback method is clear hwcodec config

Signed-off-by: 21pages <pages21@163.com>

* show current codec in menu when auto codec is chosen

Signed-off-by: 21pages <pages21@163.com>

---------

Signed-off-by: 21pages <pages21@163.com>

flutter/lib/common/widgets/toolbar.dart

@@ -355,14 +355,18 @@ Future<List<TRadioMenu<String>>> toolbarCodec(
 
   TRadioMenu<String> radio(String label, String value, bool enabled) {
     return TRadioMenu<String>(
-        child: Text(translate(label)),
+        child: Text(label),
         value: value,
         groupValue: groupValue,
         onChanged: enabled ? onChanged : null);
   }
 
+  var autoLabel = translate('Auto');
+  if (groupValue == 'auto') {
+    autoLabel = '$autoLabel (${ffi.qualityMonitorModel.data.codecFormat})';
+  }
   return [
-    radio('Auto', 'auto', true),
+    radio(autoLabel, 'auto', true),
     if (codecs[0]) radio('VP8', 'vp8', codecs[0]),
     radio('VP9', 'vp9', true),
     if (codecs[1]) radio('AV1', 'av1', codecs[1]),

libs/scrap/src/common/codec.rs

@@ -36,7 +36,7 @@ use hbb_common::{config::Config2, lazy_static};
 
 lazy_static::lazy_static! {
     static ref PEER_DECODINGS: Arc<Mutex<HashMap<i32, SupportedDecoding>>> = Default::default();
-    static ref ENCODE_CODEC_NAME: Arc<Mutex<CodecName>> = Arc::new(Mutex::new(CodecName::VP9));
+    static ref ENCODE_CODEC_FORMAT: Arc<Mutex<CodecFormat>> = Arc::new(Mutex::new(CodecFormat::VP9));
     static ref THREAD_LOG_TIME: Arc<Mutex<Option<Instant>>> = Arc::new(Mutex::new(None));
 }
 
@@ -138,7 +138,7 @@ impl Encoder {
                 Err(e) => {
                     log::error!("new hw encoder failed: {e:?}, clear config");
                     hbb_common::config::HwCodecConfig::clear_ram();
-                    *ENCODE_CODEC_NAME.lock().unwrap() = CodecName::VP9;
+                    *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
                     Err(e)
                 }
             },
@@ -150,7 +150,7 @@ impl Encoder {
                 Err(e) => {
                     log::error!("new vram encoder failed: {e:?}, clear config");
                     hbb_common::config::HwCodecConfig::clear_vram();
-                    *ENCODE_CODEC_NAME.lock().unwrap() = CodecName::VP9;
+                    *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
                     Err(e)
                 }
             },
@@ -194,20 +194,20 @@ impl Encoder {
         #[cfg(feature = "vram")]
         if enable_vram_option() {
             if _all_support_h264_decoding {
-                if VRamEncoder::available(CodecName::H264VRAM).len() > 0 {
+                if VRamEncoder::available(CodecFormat::H264).len() > 0 {
                     h264vram_encoding = true;
                 }
             }
             if _all_support_h265_decoding {
-                if VRamEncoder::available(CodecName::H265VRAM).len() > 0 {
+                if VRamEncoder::available(CodecFormat::H265).len() > 0 {
                     h265vram_encoding = true;
                 }
             }
         }
         #[allow(unused_mut)]
-        let mut h264hw_encoding = None;
+        let mut h264hw_encoding: Option<String> = None;
         #[allow(unused_mut)]
-        let mut h265hw_encoding = None;
+        let mut h265hw_encoding: Option<String> = None;
         #[cfg(feature = "hwcodec")]
         if enable_hwcodec_option() {
             if _all_support_h264_decoding {
@@ -223,7 +223,7 @@ impl Encoder {
             _all_support_h264_decoding && (h264vram_encoding || h264hw_encoding.is_some());
         let h265_useable =
             _all_support_h265_decoding && (h265vram_encoding || h265hw_encoding.is_some());
-        let mut name = ENCODE_CODEC_NAME.lock().unwrap();
+        let mut format = ENCODE_CODEC_FORMAT.lock().unwrap();
         let preferences: Vec<_> = decodings
             .iter()
             .filter(|(_, s)| {
@@ -251,32 +251,28 @@ impl Encoder {
         let preference = most_frequent.enum_value_or(PreferCodec::Auto);
 
         #[allow(unused_mut)]
-        let mut auto_codec = CodecName::VP9;
+        let mut auto_codec = CodecFormat::VP9;
         let mut system = System::new();
         system.refresh_memory();
         if vp8_useable && system.total_memory() <= 4 * 1024 * 1024 * 1024 {
             // 4 Gb
-            auto_codec = CodecName::VP8
+            auto_codec = CodecFormat::VP8
         }
 
-        *name = match preference {
+        *format = match preference {
-            PreferCodec::VP8 => CodecName::VP8,
+            PreferCodec::VP8 => CodecFormat::VP8,
-            PreferCodec::VP9 => CodecName::VP9,
+            PreferCodec::VP9 => CodecFormat::VP9,
-            PreferCodec::AV1 => CodecName::AV1,
+            PreferCodec::AV1 => CodecFormat::AV1,
             PreferCodec::H264 => {
-                if h264vram_encoding {
+                if h264vram_encoding || h264hw_encoding.is_some() {
-                    CodecName::H264VRAM
+                    CodecFormat::H264
-                } else if let Some(v) = h264hw_encoding {
-                    CodecName::H264RAM(v)
                 } else {
                     auto_codec
                 }
             }
             PreferCodec::H265 => {
-                if h265vram_encoding {
+                if h265vram_encoding || h265hw_encoding.is_some() {
-                    CodecName::H265VRAM
+                    CodecFormat::H265
-                } else if let Some(v) = h265hw_encoding {
-                    CodecName::H265RAM(v)
                 } else {
                     auto_codec
                 }
@@ -291,14 +287,14 @@ impl Encoder {
                 "connection count: {}, used preference: {:?}, encoder: {:?}",
                 decodings.len(),
                 preference,
-                *name
+                *format
             )
         }
     }
 
     #[inline]
-    pub fn negotiated_codec() -> CodecName {
+    pub fn negotiated_codec() -> CodecFormat {
-        ENCODE_CODEC_NAME.lock().unwrap().clone()
+        ENCODE_CODEC_FORMAT.lock().unwrap().clone()
     }
 
     pub fn supported_encoding() -> SupportedEncoding {
@@ -321,31 +317,31 @@ impl Encoder {
         }
         #[cfg(feature = "vram")]
         if enable_vram_option() {
-            encoding.h264 |= VRamEncoder::available(CodecName::H264VRAM).len() > 0;
+            encoding.h264 |= VRamEncoder::available(CodecFormat::H264).len() > 0;
-            encoding.h265 |= VRamEncoder::available(CodecName::H265VRAM).len() > 0;
+            encoding.h265 |= VRamEncoder::available(CodecFormat::H265).len() > 0;
         }
         encoding
     }
 
     pub fn set_fallback(config: &EncoderCfg) {
-        let name = match config {
+        let format = match config {
             EncoderCfg::VPX(vpx) => match vpx.codec {
-                VpxVideoCodecId::VP8 => CodecName::VP8,
+                VpxVideoCodecId::VP8 => CodecFormat::VP8,
-                VpxVideoCodecId::VP9 => CodecName::VP9,
+                VpxVideoCodecId::VP9 => CodecFormat::VP9,
             },
-            EncoderCfg::AOM(_) => CodecName::AV1,
+            EncoderCfg::AOM(_) => CodecFormat::AV1,
             #[cfg(feature = "hwcodec")]
             EncoderCfg::HWRAM(hw) => {
                 if hw.name.to_lowercase().contains("h264") {
-                    CodecName::H264RAM(hw.name.clone())
+                    CodecFormat::H264
                 } else {
-                    CodecName::H265RAM(hw.name.clone())
+                    CodecFormat::H265
                 }
             }
             #[cfg(feature = "vram")]
             EncoderCfg::VRAM(vram) => match vram.feature.data_format {
-                hwcodec::common::DataFormat::H264 => CodecName::H264VRAM,
+                hwcodec::common::DataFormat::H264 => CodecFormat::H264,
-                hwcodec::common::DataFormat::H265 => CodecName::H265VRAM,
+                hwcodec::common::DataFormat::H265 => CodecFormat::H265,
                 _ => {
                     log::error!(
                         "should not reach here, vram not support {:?}",
@@ -355,10 +351,10 @@ impl Encoder {
                 }
             },
         };
-        let current = ENCODE_CODEC_NAME.lock().unwrap().clone();
+        let current = ENCODE_CODEC_FORMAT.lock().unwrap().clone();
-        if current != name {
+        if current != format {
-            log::info!("codec fallback: {:?} -> {:?}", current, name);
+            log::info!("codec fallback: {:?} -> {:?}", current, format);
-            *ENCODE_CODEC_NAME.lock().unwrap() = name;
+            *ENCODE_CODEC_FORMAT.lock().unwrap() = format;
         }
     }
 

libs/scrap/src/common/vram.rs

@@ -186,8 +186,8 @@ impl EncoderApi for VRamEncoder {
 }
 
 impl VRamEncoder {
-    pub fn try_get(device: &AdapterDevice, name: CodecName) -> Option<FeatureContext> {
+    pub fn try_get(device: &AdapterDevice, format: CodecFormat) -> Option<FeatureContext> {
-        let v: Vec<_> = Self::available(name)
+        let v: Vec<_> = Self::available(format)
             .drain(..)
             .filter(|e| e.luid == device.luid)
             .collect();
@@ -202,15 +202,15 @@ impl VRamEncoder {
         }
     }
 
-    pub fn available(name: CodecName) -> Vec<FeatureContext> {
+    pub fn available(format: CodecFormat) -> Vec<FeatureContext> {
         let not_use = ENOCDE_NOT_USE.lock().unwrap().clone();
         if not_use.values().any(|not_use| *not_use) {
             log::info!("currently not use vram encoders: {not_use:?}");
             return vec![];
         }
-        let data_format = match name {
+        let data_format = match format {
-            CodecName::H264VRAM => DataFormat::H264,
+            CodecFormat::H264 => DataFormat::H264,
-            CodecName::H265VRAM => DataFormat::H265,
+            CodecFormat::H265 => DataFormat::H265,
             _ => return vec![],
         };
         let Ok(displays) = crate::Display::all() else {

src/server/video_service.rs

@@ -414,22 +414,34 @@ fn run(vs: VideoService) -> ResultType<()> {
     let record_incoming = !Config::get_option("allow-auto-record-incoming").is_empty();
     let client_record = video_qos.record();
     drop(video_qos);
-    let encoder_cfg = get_encoder_config(
+    let (mut encoder, encoder_cfg, codec_format, use_i444, recorder) = match setup_encoder(
         &c,
         display_idx,
         quality,
-        client_record || record_incoming,
+        client_record,
+        record_incoming,
         last_portable_service_running,
-    );
+    ) {
-    Encoder::set_fallback(&encoder_cfg);
+        Ok(result) => result,
-    let codec_name = Encoder::negotiated_codec();
+        Err(err) => {
-    let recorder = get_recorder(c.width, c.height, &codec_name, record_incoming);
+            log::error!("Failed to create encoder: {err:?}, fallback to VP9");
-    let mut encoder;
+            Encoder::set_fallback(&EncoderCfg::VPX(VpxEncoderConfig {
-    let use_i444 = Encoder::use_i444(&encoder_cfg);
+                width: c.width as _,
-    match Encoder::new(encoder_cfg.clone(), use_i444) {
+                height: c.height as _,
-        Ok(x) => encoder = x,
+                quality,
-        Err(err) => bail!("Failed to create encoder: {}", err),
+                codec: VpxVideoCodecId::VP9,
-    }
+                keyframe_interval: None,
+            }));
+            setup_encoder(
+                &c,
+                display_idx,
+                quality,
+                client_record,
+                record_incoming,
+                last_portable_service_running,
+            )?
+        }
+    };
     #[cfg(feature = "vram")]
     c.set_output_texture(encoder.input_texture());
     VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate());
@@ -480,7 +492,7 @@ fn run(vs: VideoService) -> ResultType<()> {
             let _ = try_broadcast_display_changed(&sp, display_idx, &c);
             bail!("SWITCH");
         }
-        if codec_name != Encoder::negotiated_codec() {
+        if codec_format != Encoder::negotiated_codec() {
             bail!("SWITCH");
         }
         #[cfg(windows)]
@@ -491,7 +503,7 @@ fn run(vs: VideoService) -> ResultType<()> {
             bail!("SWITCH");
         }
         #[cfg(all(windows, feature = "vram"))]
-        if c.is_gdi() && (codec_name == CodecName::H264VRAM || codec_name == CodecName::H265VRAM) {
+        if c.is_gdi() && encoder.input_texture() {
             log::info!("changed to gdi when using vram");
             bail!("SWITCH");
         }
@@ -630,6 +642,35 @@ impl Drop for Raii {
     }
 }
 
+fn setup_encoder(
+    c: &CapturerInfo,
+    display_idx: usize,
+    quality: Quality,
+    client_record: bool,
+    record_incoming: bool,
+    last_portable_service_running: bool,
+) -> ResultType<(
+    Encoder,
+    EncoderCfg,
+    CodecFormat,
+    bool,
+    Arc<Mutex<Option<Recorder>>>,
+)> {
+    let encoder_cfg = get_encoder_config(
+        &c,
+        display_idx,
+        quality,
+        client_record || record_incoming,
+        last_portable_service_running,
+    );
+    Encoder::set_fallback(&encoder_cfg);
+    let codec_format = Encoder::negotiated_codec();
+    let recorder = get_recorder(c.width, c.height, &codec_format, record_incoming);
+    let use_i444 = Encoder::use_i444(&encoder_cfg);
+    let encoder = Encoder::new(encoder_cfg.clone(), use_i444)?;
+    Ok((encoder, encoder_cfg, codec_format, use_i444, recorder))
+}
+
 fn get_encoder_config(
     c: &CapturerInfo,
     _display_idx: usize,
@@ -647,111 +688,57 @@ fn get_encoder_config(
     // https://www.wowza.com/community/t/the-correct-keyframe-interval-in-obs-studio/95162
     let keyframe_interval = if record { Some(240) } else { None };
     let negotiated_codec = Encoder::negotiated_codec();
-    match negotiated_codec.clone() {
+    match negotiated_codec {
-        CodecName::H264VRAM | CodecName::H265VRAM => {
+        CodecFormat::H264 | CodecFormat::H265 => {
             #[cfg(feature = "vram")]
-            if let Some(feature) = VRamEncoder::try_get(&c.device(), negotiated_codec.clone()) {
+            if let Some(feature) = VRamEncoder::try_get(&c.device(), negotiated_codec) {
-                EncoderCfg::VRAM(VRamEncoderConfig {
+                return EncoderCfg::VRAM(VRamEncoderConfig {
                     device: c.device(),
                     width: c.width,
                     height: c.height,
                     quality,
                     feature,
                     keyframe_interval,
-                })
+                });
-            } else {
-                handle_hw_encoder(
-                    negotiated_codec.clone(),
-                    c.width,
-                    c.height,
-                    quality as _,
-                    keyframe_interval,
-                )
             }
-            #[cfg(not(feature = "vram"))]
+            #[cfg(feature = "hwcodec")]
-            handle_hw_encoder(
+            if let Some(hw) = HwRamEncoder::try_get(negotiated_codec) {
-                negotiated_codec.clone(),
+                return EncoderCfg::HWRAM(HwRamEncoderConfig {
-                c.width,
+                    name: hw.name,
-                c.height,
+                    width: c.width,
-                quality as _,
+                    height: c.height,
+                    quality,
+                    keyframe_interval,
+                });
+            }
+            EncoderCfg::VPX(VpxEncoderConfig {
+                width: c.width as _,
+                height: c.height as _,
+                quality,
+                codec: VpxVideoCodecId::VP9,
                 keyframe_interval,
-            )
+            })
         }
-        CodecName::H264RAM(_name) | CodecName::H265RAM(_name) => handle_hw_encoder(
+        format @ (CodecFormat::VP8 | CodecFormat::VP9) => EncoderCfg::VPX(VpxEncoderConfig {
-            negotiated_codec.clone(),
-            c.width,
-            c.height,
-            quality as _,
-            keyframe_interval,
-        ),
-        name @ (CodecName::VP8 | CodecName::VP9) => EncoderCfg::VPX(VpxEncoderConfig {
             width: c.width as _,
             height: c.height as _,
             quality,
-            codec: if name == CodecName::VP8 {
+            codec: if format == CodecFormat::VP8 {
                 VpxVideoCodecId::VP8
             } else {
                 VpxVideoCodecId::VP9
             },
             keyframe_interval,
         }),
-        CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
+        CodecFormat::AV1 => EncoderCfg::AOM(AomEncoderConfig {
             width: c.width as _,
             height: c.height as _,
             quality,
             keyframe_interval,
         }),
-    }
-}
-
-fn handle_hw_encoder(
-    _name: CodecName,
-    width: usize,
-    height: usize,
-    quality: Quality,
-    keyframe_interval: Option<usize>,
-) -> EncoderCfg {
-    let f = || {
-        #[cfg(feature = "hwcodec")]
-        match _name {
-            CodecName::H264VRAM | CodecName::H265VRAM => {
-                let format = if _name == CodecName::H265VRAM {
-                    CodecFormat::H265
-                } else {
-                    CodecFormat::H264
-                };
-                if let Some(hw) = HwRamEncoder::try_get(format) {
-                    return Ok(EncoderCfg::HWRAM(HwRamEncoderConfig {
-                        name: hw.name,
-                        width,
-                        height,
-                        quality,
-                        keyframe_interval,
-                    }));
-                }
-            }
-            CodecName::H264RAM(name) | CodecName::H265RAM(name) => {
-                return Ok(EncoderCfg::HWRAM(HwRamEncoderConfig {
-                    name,
-                    width,
-                    height,
-                    quality,
-                    keyframe_interval,
-                }));
-            }
-            _ => {
-                return Err(());
-            }
-        };
-
-        Err(())
-    };
-
-    match f() {
-        Ok(cfg) => cfg,
         _ => EncoderCfg::VPX(VpxEncoderConfig {
-            width: width as _,
+            width: c.width as _,
-            height: height as _,
+            height: c.height as _,
             quality,
             codec: VpxVideoCodecId::VP9,
             keyframe_interval,
@@ -762,7 +749,7 @@ fn handle_hw_encoder(
 fn get_recorder(
     width: usize,
     height: usize,
-    codec_name: &CodecName,
+    codec_format: &CodecFormat,
     record_incoming: bool,
 ) -> Arc<Mutex<Option<Recorder>>> {
     let recorder = if record_incoming {
@@ -782,7 +769,7 @@ fn get_recorder(
             filename: "".to_owned(),
             width,
             height,
-            format: codec_name.into(),
+            format: codec_format.clone(),
             tx,
         })
         .map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r))))