From 3746fd88b53bce766b6e08b6a56d29aef3ca5eb3 Mon Sep 17 00:00:00 2001 From: 21pages Date: Tue, 7 May 2024 20:34:23 +0800 Subject: [PATCH] Show current codec in menu when auto codec is chosen (#7942) * change negotiated codec name to negotiated codec format Signed-off-by: 21pages * fallback to vp9 directly if failed to create encoder Current fallback method is clear hwcodec config Signed-off-by: 21pages * show current codec in menu when auto codec is chosen Signed-off-by: 21pages --------- Signed-off-by: 21pages --- flutter/lib/common/widgets/toolbar.dart | 8 +- libs/scrap/src/common/codec.rs | 74 +++++----- libs/scrap/src/common/vram.rs | 12 +- src/server/video_service.rs | 179 +++++++++++------------- 4 files changed, 130 insertions(+), 143 deletions(-) diff --git a/flutter/lib/common/widgets/toolbar.dart b/flutter/lib/common/widgets/toolbar.dart index a37111541..5999fdbf4 100644 --- a/flutter/lib/common/widgets/toolbar.dart +++ b/flutter/lib/common/widgets/toolbar.dart @@ -355,14 +355,18 @@ Future>> toolbarCodec( TRadioMenu radio(String label, String value, bool enabled) { return TRadioMenu( - child: Text(translate(label)), + child: Text(label), value: value, groupValue: groupValue, onChanged: enabled ? onChanged : null); } + var autoLabel = translate('Auto'); + if (groupValue == 'auto') { + autoLabel = '$autoLabel (${ffi.qualityMonitorModel.data.codecFormat})'; + } return [ - radio('Auto', 'auto', true), + radio(autoLabel, 'auto', true), if (codecs[0]) radio('VP8', 'vp8', codecs[0]), radio('VP9', 'vp9', true), if (codecs[1]) radio('AV1', 'av1', codecs[1]), diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 2c7690e2c..2c383b99a 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -36,7 +36,7 @@ use hbb_common::{config::Config2, lazy_static}; lazy_static::lazy_static! { static ref PEER_DECODINGS: Arc>> = Default::default(); - static ref ENCODE_CODEC_NAME: Arc> = Arc::new(Mutex::new(CodecName::VP9)); + static ref ENCODE_CODEC_FORMAT: Arc> = Arc::new(Mutex::new(CodecFormat::VP9)); static ref THREAD_LOG_TIME: Arc>> = Arc::new(Mutex::new(None)); } @@ -138,7 +138,7 @@ impl Encoder { Err(e) => { log::error!("new hw encoder failed: {e:?}, clear config"); hbb_common::config::HwCodecConfig::clear_ram(); - *ENCODE_CODEC_NAME.lock().unwrap() = CodecName::VP9; + *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9; Err(e) } }, @@ -150,7 +150,7 @@ impl Encoder { Err(e) => { log::error!("new vram encoder failed: {e:?}, clear config"); hbb_common::config::HwCodecConfig::clear_vram(); - *ENCODE_CODEC_NAME.lock().unwrap() = CodecName::VP9; + *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9; Err(e) } }, @@ -194,20 +194,20 @@ impl Encoder { #[cfg(feature = "vram")] if enable_vram_option() { if _all_support_h264_decoding { - if VRamEncoder::available(CodecName::H264VRAM).len() > 0 { + if VRamEncoder::available(CodecFormat::H264).len() > 0 { h264vram_encoding = true; } } if _all_support_h265_decoding { - if VRamEncoder::available(CodecName::H265VRAM).len() > 0 { + if VRamEncoder::available(CodecFormat::H265).len() > 0 { h265vram_encoding = true; } } } #[allow(unused_mut)] - let mut h264hw_encoding = None; + let mut h264hw_encoding: Option = None; #[allow(unused_mut)] - let mut h265hw_encoding = None; + let mut h265hw_encoding: Option = None; #[cfg(feature = "hwcodec")] if enable_hwcodec_option() { if _all_support_h264_decoding { @@ -223,7 +223,7 @@ impl Encoder { _all_support_h264_decoding && (h264vram_encoding || h264hw_encoding.is_some()); let h265_useable = _all_support_h265_decoding && (h265vram_encoding || h265hw_encoding.is_some()); - let mut name = ENCODE_CODEC_NAME.lock().unwrap(); + let mut format = ENCODE_CODEC_FORMAT.lock().unwrap(); let preferences: Vec<_> = decodings .iter() .filter(|(_, s)| { @@ -251,32 +251,28 @@ impl Encoder { let preference = most_frequent.enum_value_or(PreferCodec::Auto); #[allow(unused_mut)] - let mut auto_codec = CodecName::VP9; + let mut auto_codec = CodecFormat::VP9; let mut system = System::new(); system.refresh_memory(); if vp8_useable && system.total_memory() <= 4 * 1024 * 1024 * 1024 { // 4 Gb - auto_codec = CodecName::VP8 + auto_codec = CodecFormat::VP8 } - *name = match preference { - PreferCodec::VP8 => CodecName::VP8, - PreferCodec::VP9 => CodecName::VP9, - PreferCodec::AV1 => CodecName::AV1, + *format = match preference { + PreferCodec::VP8 => CodecFormat::VP8, + PreferCodec::VP9 => CodecFormat::VP9, + PreferCodec::AV1 => CodecFormat::AV1, PreferCodec::H264 => { - if h264vram_encoding { - CodecName::H264VRAM - } else if let Some(v) = h264hw_encoding { - CodecName::H264RAM(v) + if h264vram_encoding || h264hw_encoding.is_some() { + CodecFormat::H264 } else { auto_codec } } PreferCodec::H265 => { - if h265vram_encoding { - CodecName::H265VRAM - } else if let Some(v) = h265hw_encoding { - CodecName::H265RAM(v) + if h265vram_encoding || h265hw_encoding.is_some() { + CodecFormat::H265 } else { auto_codec } @@ -291,14 +287,14 @@ impl Encoder { "connection count: {}, used preference: {:?}, encoder: {:?}", decodings.len(), preference, - *name + *format ) } } #[inline] - pub fn negotiated_codec() -> CodecName { - ENCODE_CODEC_NAME.lock().unwrap().clone() + pub fn negotiated_codec() -> CodecFormat { + ENCODE_CODEC_FORMAT.lock().unwrap().clone() } pub fn supported_encoding() -> SupportedEncoding { @@ -321,31 +317,31 @@ impl Encoder { } #[cfg(feature = "vram")] if enable_vram_option() { - encoding.h264 |= VRamEncoder::available(CodecName::H264VRAM).len() > 0; - encoding.h265 |= VRamEncoder::available(CodecName::H265VRAM).len() > 0; + encoding.h264 |= VRamEncoder::available(CodecFormat::H264).len() > 0; + encoding.h265 |= VRamEncoder::available(CodecFormat::H265).len() > 0; } encoding } pub fn set_fallback(config: &EncoderCfg) { - let name = match config { + let format = match config { EncoderCfg::VPX(vpx) => match vpx.codec { - VpxVideoCodecId::VP8 => CodecName::VP8, - VpxVideoCodecId::VP9 => CodecName::VP9, + VpxVideoCodecId::VP8 => CodecFormat::VP8, + VpxVideoCodecId::VP9 => CodecFormat::VP9, }, - EncoderCfg::AOM(_) => CodecName::AV1, + EncoderCfg::AOM(_) => CodecFormat::AV1, #[cfg(feature = "hwcodec")] EncoderCfg::HWRAM(hw) => { if hw.name.to_lowercase().contains("h264") { - CodecName::H264RAM(hw.name.clone()) + CodecFormat::H264 } else { - CodecName::H265RAM(hw.name.clone()) + CodecFormat::H265 } } #[cfg(feature = "vram")] EncoderCfg::VRAM(vram) => match vram.feature.data_format { - hwcodec::common::DataFormat::H264 => CodecName::H264VRAM, - hwcodec::common::DataFormat::H265 => CodecName::H265VRAM, + hwcodec::common::DataFormat::H264 => CodecFormat::H264, + hwcodec::common::DataFormat::H265 => CodecFormat::H265, _ => { log::error!( "should not reach here, vram not support {:?}", @@ -355,10 +351,10 @@ impl Encoder { } }, }; - let current = ENCODE_CODEC_NAME.lock().unwrap().clone(); - if current != name { - log::info!("codec fallback: {:?} -> {:?}", current, name); - *ENCODE_CODEC_NAME.lock().unwrap() = name; + let current = ENCODE_CODEC_FORMAT.lock().unwrap().clone(); + if current != format { + log::info!("codec fallback: {:?} -> {:?}", current, format); + *ENCODE_CODEC_FORMAT.lock().unwrap() = format; } } diff --git a/libs/scrap/src/common/vram.rs b/libs/scrap/src/common/vram.rs index 05295c824..0a8b5c2d0 100644 --- a/libs/scrap/src/common/vram.rs +++ b/libs/scrap/src/common/vram.rs @@ -186,8 +186,8 @@ impl EncoderApi for VRamEncoder { } impl VRamEncoder { - pub fn try_get(device: &AdapterDevice, name: CodecName) -> Option { - let v: Vec<_> = Self::available(name) + pub fn try_get(device: &AdapterDevice, format: CodecFormat) -> Option { + let v: Vec<_> = Self::available(format) .drain(..) .filter(|e| e.luid == device.luid) .collect(); @@ -202,15 +202,15 @@ impl VRamEncoder { } } - pub fn available(name: CodecName) -> Vec { + pub fn available(format: CodecFormat) -> Vec { let not_use = ENOCDE_NOT_USE.lock().unwrap().clone(); if not_use.values().any(|not_use| *not_use) { log::info!("currently not use vram encoders: {not_use:?}"); return vec![]; } - let data_format = match name { - CodecName::H264VRAM => DataFormat::H264, - CodecName::H265VRAM => DataFormat::H265, + let data_format = match format { + CodecFormat::H264 => DataFormat::H264, + CodecFormat::H265 => DataFormat::H265, _ => return vec![], }; let Ok(displays) = crate::Display::all() else { diff --git a/src/server/video_service.rs b/src/server/video_service.rs index e5db17f93..3381fa709 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -414,22 +414,34 @@ fn run(vs: VideoService) -> ResultType<()> { let record_incoming = !Config::get_option("allow-auto-record-incoming").is_empty(); let client_record = video_qos.record(); drop(video_qos); - let encoder_cfg = get_encoder_config( + let (mut encoder, encoder_cfg, codec_format, use_i444, recorder) = match setup_encoder( &c, display_idx, quality, - client_record || record_incoming, + client_record, + record_incoming, last_portable_service_running, - ); - Encoder::set_fallback(&encoder_cfg); - let codec_name = Encoder::negotiated_codec(); - let recorder = get_recorder(c.width, c.height, &codec_name, record_incoming); - let mut encoder; - let use_i444 = Encoder::use_i444(&encoder_cfg); - match Encoder::new(encoder_cfg.clone(), use_i444) { - Ok(x) => encoder = x, - Err(err) => bail!("Failed to create encoder: {}", err), - } + ) { + Ok(result) => result, + Err(err) => { + log::error!("Failed to create encoder: {err:?}, fallback to VP9"); + Encoder::set_fallback(&EncoderCfg::VPX(VpxEncoderConfig { + width: c.width as _, + height: c.height as _, + quality, + codec: VpxVideoCodecId::VP9, + keyframe_interval: None, + })); + setup_encoder( + &c, + display_idx, + quality, + client_record, + record_incoming, + last_portable_service_running, + )? + } + }; #[cfg(feature = "vram")] c.set_output_texture(encoder.input_texture()); VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate()); @@ -480,7 +492,7 @@ fn run(vs: VideoService) -> ResultType<()> { let _ = try_broadcast_display_changed(&sp, display_idx, &c); bail!("SWITCH"); } - if codec_name != Encoder::negotiated_codec() { + if codec_format != Encoder::negotiated_codec() { bail!("SWITCH"); } #[cfg(windows)] @@ -491,7 +503,7 @@ fn run(vs: VideoService) -> ResultType<()> { bail!("SWITCH"); } #[cfg(all(windows, feature = "vram"))] - if c.is_gdi() && (codec_name == CodecName::H264VRAM || codec_name == CodecName::H265VRAM) { + if c.is_gdi() && encoder.input_texture() { log::info!("changed to gdi when using vram"); bail!("SWITCH"); } @@ -630,6 +642,35 @@ impl Drop for Raii { } } +fn setup_encoder( + c: &CapturerInfo, + display_idx: usize, + quality: Quality, + client_record: bool, + record_incoming: bool, + last_portable_service_running: bool, +) -> ResultType<( + Encoder, + EncoderCfg, + CodecFormat, + bool, + Arc>>, +)> { + let encoder_cfg = get_encoder_config( + &c, + display_idx, + quality, + client_record || record_incoming, + last_portable_service_running, + ); + Encoder::set_fallback(&encoder_cfg); + let codec_format = Encoder::negotiated_codec(); + let recorder = get_recorder(c.width, c.height, &codec_format, record_incoming); + let use_i444 = Encoder::use_i444(&encoder_cfg); + let encoder = Encoder::new(encoder_cfg.clone(), use_i444)?; + Ok((encoder, encoder_cfg, codec_format, use_i444, recorder)) +} + fn get_encoder_config( c: &CapturerInfo, _display_idx: usize, @@ -647,111 +688,57 @@ fn get_encoder_config( // https://www.wowza.com/community/t/the-correct-keyframe-interval-in-obs-studio/95162 let keyframe_interval = if record { Some(240) } else { None }; let negotiated_codec = Encoder::negotiated_codec(); - match negotiated_codec.clone() { - CodecName::H264VRAM | CodecName::H265VRAM => { + match negotiated_codec { + CodecFormat::H264 | CodecFormat::H265 => { #[cfg(feature = "vram")] - if let Some(feature) = VRamEncoder::try_get(&c.device(), negotiated_codec.clone()) { - EncoderCfg::VRAM(VRamEncoderConfig { + if let Some(feature) = VRamEncoder::try_get(&c.device(), negotiated_codec) { + return EncoderCfg::VRAM(VRamEncoderConfig { device: c.device(), width: c.width, height: c.height, quality, feature, keyframe_interval, - }) - } else { - handle_hw_encoder( - negotiated_codec.clone(), - c.width, - c.height, - quality as _, - keyframe_interval, - ) + }); } - #[cfg(not(feature = "vram"))] - handle_hw_encoder( - negotiated_codec.clone(), - c.width, - c.height, - quality as _, + #[cfg(feature = "hwcodec")] + if let Some(hw) = HwRamEncoder::try_get(negotiated_codec) { + return EncoderCfg::HWRAM(HwRamEncoderConfig { + name: hw.name, + width: c.width, + height: c.height, + quality, + keyframe_interval, + }); + } + EncoderCfg::VPX(VpxEncoderConfig { + width: c.width as _, + height: c.height as _, + quality, + codec: VpxVideoCodecId::VP9, keyframe_interval, - ) + }) } - CodecName::H264RAM(_name) | CodecName::H265RAM(_name) => handle_hw_encoder( - negotiated_codec.clone(), - c.width, - c.height, - quality as _, - keyframe_interval, - ), - name @ (CodecName::VP8 | CodecName::VP9) => EncoderCfg::VPX(VpxEncoderConfig { + format @ (CodecFormat::VP8 | CodecFormat::VP9) => EncoderCfg::VPX(VpxEncoderConfig { width: c.width as _, height: c.height as _, quality, - codec: if name == CodecName::VP8 { + codec: if format == CodecFormat::VP8 { VpxVideoCodecId::VP8 } else { VpxVideoCodecId::VP9 }, keyframe_interval, }), - CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig { + CodecFormat::AV1 => EncoderCfg::AOM(AomEncoderConfig { width: c.width as _, height: c.height as _, quality, keyframe_interval, }), - } -} - -fn handle_hw_encoder( - _name: CodecName, - width: usize, - height: usize, - quality: Quality, - keyframe_interval: Option, -) -> EncoderCfg { - let f = || { - #[cfg(feature = "hwcodec")] - match _name { - CodecName::H264VRAM | CodecName::H265VRAM => { - let format = if _name == CodecName::H265VRAM { - CodecFormat::H265 - } else { - CodecFormat::H264 - }; - if let Some(hw) = HwRamEncoder::try_get(format) { - return Ok(EncoderCfg::HWRAM(HwRamEncoderConfig { - name: hw.name, - width, - height, - quality, - keyframe_interval, - })); - } - } - CodecName::H264RAM(name) | CodecName::H265RAM(name) => { - return Ok(EncoderCfg::HWRAM(HwRamEncoderConfig { - name, - width, - height, - quality, - keyframe_interval, - })); - } - _ => { - return Err(()); - } - }; - - Err(()) - }; - - match f() { - Ok(cfg) => cfg, _ => EncoderCfg::VPX(VpxEncoderConfig { - width: width as _, - height: height as _, + width: c.width as _, + height: c.height as _, quality, codec: VpxVideoCodecId::VP9, keyframe_interval, @@ -762,7 +749,7 @@ fn handle_hw_encoder( fn get_recorder( width: usize, height: usize, - codec_name: &CodecName, + codec_format: &CodecFormat, record_incoming: bool, ) -> Arc>> { let recorder = if record_incoming { @@ -782,7 +769,7 @@ fn get_recorder( filename: "".to_owned(), width, height, - format: codec_name.into(), + format: codec_format.clone(), tx, }) .map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r))))