diff --git a/flutter/lib/desktop/pages/desktop_setting_page.dart b/flutter/lib/desktop/pages/desktop_setting_page.dart index 74d51407c..66ef83d31 100644 --- a/flutter/lib/desktop/pages/desktop_setting_page.dart +++ b/flutter/lib/desktop/pages/desktop_setting_page.dart @@ -1258,6 +1258,9 @@ class _DisplayState extends State<_Display> { } Widget codec(BuildContext context) { + if (!bind.mainHasHwcodec()) { + return Offstage(); + } final key = 'codec-preference'; onChanged(String value) async { await bind.mainSetUserDefaultOption(key: key, value: value); @@ -1265,45 +1268,28 @@ class _DisplayState extends State<_Display> { } final groupValue = bind.mainGetUserDefaultOption(key: key); - var hwRadios = []; - try { - final Map codecsJson = jsonDecode(bind.mainSupportedHwdecodings()); - final h264 = codecsJson['h264'] ?? false; - final h265 = codecsJson['h265'] ?? false; - if (h264) { - hwRadios.add(_Radio(context, - value: 'h264', - groupValue: groupValue, - label: 'H264', - onChanged: onChanged)); - } - if (h265) { - hwRadios.add(_Radio(context, - value: 'h265', - groupValue: groupValue, - label: 'H265', - onChanged: onChanged)); - } - } catch (e) { - debugPrint("failed to parse supported hwdecodings, err=$e"); - } + return _Card(title: 'Default Codec', children: [ _Radio(context, value: 'auto', groupValue: groupValue, label: 'Auto', onChanged: onChanged), - _Radio(context, - value: 'vp8', - groupValue: groupValue, - label: 'VP8', - onChanged: onChanged), _Radio(context, value: 'vp9', groupValue: groupValue, label: 'VP9', onChanged: onChanged), - ...hwRadios, + _Radio(context, + value: 'h264', + groupValue: groupValue, + label: 'H264', + onChanged: onChanged), + _Radio(context, + value: 'h265', + groupValue: groupValue, + label: 'H265', + onChanged: onChanged), ]); } diff --git a/flutter/lib/desktop/widgets/remote_toolbar.dart b/flutter/lib/desktop/widgets/remote_toolbar.dart index cb4b6d644..0ef8674ef 100644 --- a/flutter/lib/desktop/widgets/remote_toolbar.dart +++ b/flutter/lib/desktop/widgets/remote_toolbar.dart @@ -1349,30 +1349,29 @@ class _DisplayMenuState extends State<_DisplayMenu> { codec() { return futureBuilder(future: () async { - final alternativeCodecs = - await bind.sessionAlternativeCodecs(id: widget.id); + final supportedHwcodec = + await bind.sessionSupportedHwcodec(id: widget.id); final codecPreference = await bind.sessionGetOption(id: widget.id, arg: 'codec-preference') ?? ''; return { - 'alternativeCodecs': alternativeCodecs, + 'supportedHwcodec': supportedHwcodec, 'codecPreference': codecPreference }; }(), hasData: (data) { final List codecs = []; try { - final Map codecsJson = jsonDecode(data['alternativeCodecs']); - final vp8 = codecsJson['vp8'] ?? false; + final Map codecsJson = jsonDecode(data['supportedHwcodec']); final h264 = codecsJson['h264'] ?? false; final h265 = codecsJson['h265'] ?? false; - codecs.add(vp8); codecs.add(h264); codecs.add(h265); } catch (e) { debugPrint("Show Codec Preference err=$e"); } - final visible = - codecs.length == 3 && (codecs[0] || codecs[1] || codecs[2]); + final visible = bind.mainHasHwcodec() && + codecs.length == 2 && + (codecs[0] || codecs[1]); if (!visible) return Offstage(); final groupValue = data['codecPreference'] as String; onChanged(String? value) async { @@ -1393,13 +1392,6 @@ class _DisplayMenuState extends State<_DisplayMenu> { onChanged: onChanged, ffi: widget.ffi, ), - _RadioMenuButton( - child: Text(translate('VP8')), - value: 'vp8', - groupValue: groupValue, - onChanged: codecs[0] ? onChanged : null, - ffi: widget.ffi, - ), _RadioMenuButton( child: Text(translate('VP9')), value: 'vp9', @@ -1411,14 +1403,14 @@ class _DisplayMenuState extends State<_DisplayMenu> { child: Text(translate('H264')), value: 'h264', groupValue: groupValue, - onChanged: codecs[1] ? onChanged : null, + onChanged: codecs[0] ? onChanged : null, ffi: widget.ffi, ), _RadioMenuButton( child: Text(translate('H265')), value: 'h265', groupValue: groupValue, - onChanged: codecs[2] ? onChanged : null, + onChanged: codecs[1] ? onChanged : null, ffi: widget.ffi, ), ]); diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index 74a327c27..083cdcd1c 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -973,11 +973,9 @@ void showOptions( if (hasHwcodec) { try { final Map codecsJson = - jsonDecode(await bind.sessionAlternativeCodecs(id: id)); - final vp8 = codecsJson['vp8'] ?? false; + jsonDecode(await bind.sessionSupportedHwcodec(id: id)); final h264 = codecsJson['h264'] ?? false; final h265 = codecsJson['h265'] ?? false; - codecs.add(vp8); codecs.add(h264); codecs.add(h265); } catch (e) { @@ -1046,7 +1044,6 @@ void showOptions( if (hasHwcodec && codecs.length == 2 && (codecs[0] || codecs[1])) { radios.addAll([ getRadio(translate('Auto'), 'auto', codec, setCodec), - getRadio('VP8', 'vp8', codec, setCodec), getRadio('VP9', 'vp9', codec, setCodec), ]); if (codecs[0]) { diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 321747e3a..6295c160b 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -24,7 +24,6 @@ message VideoFrame { YUV yuv = 8; EncodedVideoFrames h264s = 10; EncodedVideoFrames h265s = 11; - EncodedVideoFrames vp8s = 12; } } @@ -77,7 +76,6 @@ message Features { message SupportedEncoding { bool h264 = 1; bool h265 = 2; - bool vp8 = 3; } message PeerInfo { @@ -459,20 +457,18 @@ enum ImageQuality { Best = 4; } -message SupportedDecoding { +message VideoCodecState { enum PreferCodec { Auto = 0; - VP9 = 1; + VPX = 1; H264 = 2; H265 = 3; - VP8 = 4; } - int32 ability_vp9 = 1; - int32 ability_h264 = 2; - int32 ability_h265 = 3; + int32 score_vpx = 1; + int32 score_h264 = 2; + int32 score_h265 = 3; PreferCodec prefer = 4; - int32 ability_vp8 = 5; } message OptionMessage { @@ -490,7 +486,7 @@ message OptionMessage { BoolOption disable_audio = 7; BoolOption disable_clipboard = 8; BoolOption enable_file_transfer = 9; - SupportedDecoding supported_decoding = 10; + VideoCodecState video_codec_state = 10; int32 custom_fps = 11; BoolOption disable_keyboard = 12; } diff --git a/libs/hbb_common/src/config.rs b/libs/hbb_common/src/config.rs index 960074a8f..6a823c7b7 100644 --- a/libs/hbb_common/src/config.rs +++ b/libs/hbb_common/src/config.rs @@ -917,8 +917,7 @@ impl PeerConfig { store = store || store2; for opt in ["rdp_password", "os-password"] { if let Some(v) = config.options.get_mut(opt) { - let (encrypted, _, store2) = - decrypt_str_or_original(v, PASSWORD_ENC_VERSION); + let (encrypted, _, store2) = decrypt_str_or_original(v, PASSWORD_ENC_VERSION); *v = encrypted; store = store || store2; } @@ -1357,7 +1356,7 @@ impl UserDefaultConfig { "view_style" => self.get_string(key, "original", vec!["adaptive"]), "scroll_style" => self.get_string(key, "scrollauto", vec!["scrollbar"]), "image_quality" => self.get_string(key, "balanced", vec!["best", "low", "custom"]), - "codec-preference" => self.get_string(key, "auto", vec!["vp8", "vp9", "h264", "h265"]), + "codec-preference" => self.get_string(key, "auto", vec!["vp9", "h264", "h265"]), "custom_image_quality" => self.get_double_string(key, 50.0, 10.0, 100.0), "custom-fps" => self.get_double_string(key, 30.0, 10.0, 120.0), _ => self diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index ba8dec9f2..003830f95 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -3,8 +3,7 @@ use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV}; use scrap::{ codec::{EncoderApi, EncoderCfg}, Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, - VpxVideoCodecId::{self, *}, - STRIDE_ALIGN, + VpxVideoCodecId, STRIDE_ALIGN, }; use std::{io::Write, time::Instant}; @@ -50,7 +49,7 @@ fn main() { "benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}", width, height, bitrate_k, args.flag_hw_pixfmt ); - [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count)); + test_vp9(&yuvs, width, height, bitrate_k, yuv_count); #[cfg(feature = "hwcodec")] { use hwcodec::AVPixelFormat; @@ -58,7 +57,7 @@ fn main() { Pixfmt::I420 => AVPixelFormat::AV_PIX_FMT_YUV420P, Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12, }; - let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt); + let yuvs = hw::vp9_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt); hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt); } } @@ -88,20 +87,13 @@ fn capture_yuv(yuv_count: usize) -> (Vec>, usize, usize) { } } -fn test_vpx( - codec_id: VpxVideoCodecId, - yuvs: &Vec>, - width: usize, - height: usize, - bitrate_k: usize, - yuv_count: usize, -) { +fn test_vp9(yuvs: &Vec>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) { let config = EncoderCfg::VPX(VpxEncoderConfig { width: width as _, height: height as _, timebase: [1, 1000], bitrate: bitrate_k as _, - codec: codec_id, + codec: VpxVideoCodecId::VP9, num_threads: (num_cpus::get() / 2) as _, }); let mut encoder = VpxEncoder::new(config).unwrap(); @@ -112,43 +104,35 @@ fn test_vpx( .unwrap(); let _ = encoder.flush().unwrap(); } - println!( - "{:?} encode: {:?}", - codec_id, - start.elapsed() / yuv_count as _ - ); + println!("vp9 encode: {:?}", start.elapsed() / yuv_count as _); // prepare data separately - let mut vpxs = vec![]; + let mut vp9s = vec![]; let start = Instant::now(); for yuv in yuvs { for ref frame in encoder .encode(start.elapsed().as_millis() as _, yuv, STRIDE_ALIGN) .unwrap() { - vpxs.push(frame.data.to_vec()); + vp9s.push(frame.data.to_vec()); } for ref frame in encoder.flush().unwrap() { - vpxs.push(frame.data.to_vec()); + vp9s.push(frame.data.to_vec()); } } - assert_eq!(vpxs.len(), yuv_count); + assert_eq!(vp9s.len(), yuv_count); let mut decoder = VpxDecoder::new(VpxDecoderConfig { - codec: codec_id, + codec: VpxVideoCodecId::VP9, num_threads: (num_cpus::get() / 2) as _, }) .unwrap(); let start = Instant::now(); - for vpx in vpxs { - let _ = decoder.decode(&vpx); + for vp9 in vp9s { + let _ = decoder.decode(&vp9); let _ = decoder.flush(); } - println!( - "{:?} decode: {:?}", - codec_id, - start.elapsed() / yuv_count as _ - ); + println!("vp9 decode: {:?}", start.elapsed() / yuv_count as _); } #[cfg(feature = "hwcodec")] @@ -283,7 +267,7 @@ mod hw { Some(info.clone()) == best.h264 || Some(info.clone()) == best.h265 } - pub fn vpx_yuv_to_hw_yuv( + pub fn vp9_yuv_to_hw_yuv( yuvs: Vec>, width: usize, height: usize, diff --git a/libs/scrap/src/common/android.rs b/libs/scrap/src/common/android.rs index 36d6a8a9b..8daf8e4bb 100644 --- a/libs/scrap/src/common/android.rs +++ b/libs/scrap/src/common/android.rs @@ -50,7 +50,6 @@ impl crate::TraitCapturer for Capturer { pub enum Frame<'a> { RAW(&'a [u8]), - VP8(&'a [u8]), VP9(&'a [u8]), Empty, } diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 3209933b4..9e4b6fce4 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -1,6 +1,7 @@ +use std::ops::{Deref, DerefMut}; +#[cfg(feature = "hwcodec")] use std::{ collections::HashMap, - ops::{Deref, DerefMut}, sync::{Arc, Mutex}, }; @@ -10,31 +11,30 @@ use crate::hwcodec::*; use crate::mediacodec::{ MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT, }; -use crate::{vpxcodec::*, CodecName, ImageFormat}; +use crate::{vpxcodec::*, ImageFormat}; -#[cfg(not(any(target_os = "android", target_os = "ios")))] -use hbb_common::sysinfo::{System, SystemExt}; use hbb_common::{ anyhow::anyhow, - config::PeerConfig, log, - message_proto::{ - supported_decoding::PreferCodec, video_frame, EncodedVideoFrames, Message, - SupportedDecoding, SupportedEncoding, - }, + message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState}, ResultType, }; #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] -use hbb_common::{config::Config2, lazy_static}; +use hbb_common::{ + config::{Config2, PeerConfig}, + lazy_static, + message_proto::video_codec_state::PreferCodec, +}; +#[cfg(feature = "hwcodec")] lazy_static::lazy_static! { - static ref PEER_DECODINGS: Arc>> = Default::default(); - static ref CODEC_NAME: Arc> = Arc::new(Mutex::new(CodecName::VP9)); + static ref PEER_DECODER_STATES: Arc>> = Default::default(); } +const SCORE_VPX: i32 = 90; #[derive(Debug, Clone)] pub struct HwEncoderConfig { - pub name: String, + pub codec_name: String, pub width: usize, pub height: usize, pub bitrate: i32, @@ -58,6 +58,10 @@ pub trait EncoderApi { fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>; } +pub struct DecoderCfg { + pub vpx: VpxDecoderConfig, +} + pub struct Encoder { pub codec: Box, } @@ -77,8 +81,7 @@ impl DerefMut for Encoder { } pub struct Decoder { - vp8: VpxDecoder, - vp9: VpxDecoder, + vpx: VpxDecoder, #[cfg(feature = "hwcodec")] hw: HwDecoders, #[cfg(feature = "hwcodec")] @@ -88,10 +91,10 @@ pub struct Decoder { } #[derive(Debug, Clone)] -pub enum EncodingUpdate { - New(SupportedDecoding), +pub enum EncoderUpdate { + State(VideoCodecState), Remove, - NewOnlyVP9, + DisableHwIfNotExist, } impl Encoder { @@ -117,156 +120,172 @@ impl Encoder { } } - pub fn update(id: i32, update: EncodingUpdate) { - let mut decodings = PEER_DECODINGS.lock().unwrap(); - match update { - EncodingUpdate::New(decoding) => { - decodings.insert(id, decoding); - } - EncodingUpdate::Remove => { - decodings.remove(&id); - } - EncodingUpdate::NewOnlyVP9 => { - decodings.insert( - id, - SupportedDecoding { - ability_vp9: 1, - ..Default::default() - }, - ); - } - } - - let vp8_useable = decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_vp8 > 0); - #[allow(unused_mut)] - let mut h264_name = None; - #[allow(unused_mut)] - let mut h265_name = None; + // TODO + pub fn update_video_encoder(id: i32, update: EncoderUpdate) { #[cfg(feature = "hwcodec")] { - let best = HwEncoder::best(); - let h264_useable = - decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_h264 > 0); - let h265_useable = - decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_h265 > 0); - if h264_useable { - h264_name = best.h264.map_or(None, |c| Some(c.name)); + let mut states = PEER_DECODER_STATES.lock().unwrap(); + match update { + EncoderUpdate::State(state) => { + states.insert(id, state); + } + EncoderUpdate::Remove => { + states.remove(&id); + } + EncoderUpdate::DisableHwIfNotExist => { + if !states.contains_key(&id) { + states.insert(id, VideoCodecState::default()); + } + } } - if h265_useable { - h265_name = best.h265.map_or(None, |c| Some(c.name)); + let name = HwEncoder::current_name(); + if states.len() > 0 { + let best = HwEncoder::best(); + let enabled_h264 = best.h264.is_some() + && states.len() > 0 + && states.iter().all(|(_, s)| s.score_h264 > 0); + let enabled_h265 = best.h265.is_some() + && states.len() > 0 + && states.iter().all(|(_, s)| s.score_h265 > 0); + + // Preference first + let mut preference = PreferCodec::Auto; + let preferences: Vec<_> = states + .iter() + .filter(|(_, s)| { + s.prefer == PreferCodec::VPX.into() + || s.prefer == PreferCodec::H264.into() && enabled_h264 + || s.prefer == PreferCodec::H265.into() && enabled_h265 + }) + .map(|(_, s)| s.prefer) + .collect(); + if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) { + preference = preferences[0].enum_value_or(PreferCodec::Auto); + } + + match preference { + PreferCodec::VPX => *name.lock().unwrap() = None, + PreferCodec::H264 => { + *name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name)) + } + PreferCodec::H265 => { + *name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name)) + } + PreferCodec::Auto => { + // score encoder + let mut score_vpx = SCORE_VPX; + let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score); + let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score); + + // score decoder + score_vpx += states.iter().map(|s| s.1.score_vpx).sum::(); + if enabled_h264 { + score_h264 += states.iter().map(|s| s.1.score_h264).sum::(); + } + if enabled_h265 { + score_h265 += states.iter().map(|s| s.1.score_h265).sum::(); + } + + if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { + *name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name)); + } else if enabled_h264 + && score_h264 >= score_vpx + && score_h264 >= score_h265 + { + *name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name)); + } else { + *name.lock().unwrap() = None; + } + } + } + + log::info!( + "connection count:{}, used preference:{:?}, encoder:{:?}", + states.len(), + preference, + name.lock().unwrap() + ) + } else { + *name.lock().unwrap() = None; } } - - let mut name = CODEC_NAME.lock().unwrap(); - let mut preference = PreferCodec::Auto; - let preferences: Vec<_> = decodings - .iter() - .filter(|(_, s)| { - s.prefer == PreferCodec::VP9.into() - || s.prefer == PreferCodec::VP8.into() && vp8_useable - || s.prefer == PreferCodec::H264.into() && h264_name.is_some() - || s.prefer == PreferCodec::H265.into() && h265_name.is_some() - }) - .map(|(_, s)| s.prefer) - .collect(); - if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) { - preference = preferences[0].enum_value_or(PreferCodec::Auto); + #[cfg(not(feature = "hwcodec"))] + { + let _ = id; + let _ = update; } - - #[allow(unused_mut)] - let mut auto_codec = CodecName::VP9; - #[cfg(not(any(target_os = "android", target_os = "ios")))] - if vp8_useable && System::new_all().total_memory() <= 4 * 1024 * 1024 * 1024 { - // 4 Gb - auto_codec = CodecName::VP8 - } - - match preference { - PreferCodec::VP8 => *name = CodecName::VP8, - PreferCodec::VP9 => *name = CodecName::VP9, - PreferCodec::H264 => *name = h264_name.map_or(auto_codec, |c| CodecName::H264(c)), - PreferCodec::H265 => *name = h265_name.map_or(auto_codec, |c| CodecName::H265(c)), - PreferCodec::Auto => *name = auto_codec, - } - - log::info!( - "connection count:{}, used preference:{:?}, encoder:{:?}", - decodings.len(), - preference, - *name - ) } - #[inline] - pub fn negotiated_codec() -> CodecName { - CODEC_NAME.lock().unwrap().clone() + pub fn current_hw_encoder_name() -> Option { + #[cfg(feature = "hwcodec")] + if enable_hwcodec_option() { + return HwEncoder::current_name().lock().unwrap().clone(); + } else { + return None; + } + #[cfg(not(feature = "hwcodec"))] + return None; } - pub fn supported_encoding() -> SupportedEncoding { - #[allow(unused_mut)] - let mut encoding = SupportedEncoding { - vp8: true, - ..Default::default() - }; + pub fn supported_encoding() -> (bool, bool) { #[cfg(feature = "hwcodec")] if enable_hwcodec_option() { let best = HwEncoder::best(); - encoding.h264 = best.h264.is_some(); - encoding.h265 = best.h265.is_some(); + ( + best.h264.as_ref().map_or(false, |c| c.score > 0), + best.h265.as_ref().map_or(false, |c| c.score > 0), + ) + } else { + (false, false) } - encoding + #[cfg(not(feature = "hwcodec"))] + (false, false) } } impl Decoder { - pub fn supported_decodings(id_for_perfer: Option<&str>) -> SupportedDecoding { - #[allow(unused_mut)] - let mut decoding = SupportedDecoding { - ability_vp8: 1, - ability_vp9: 1, - prefer: id_for_perfer - .map_or(PreferCodec::Auto, |id| Self::codec_preference(id)) - .into(), - ..Default::default() - }; + pub fn video_codec_state(_id: &str) -> VideoCodecState { #[cfg(feature = "hwcodec")] if enable_hwcodec_option() { let best = HwDecoder::best(); - decoding.ability_h264 = if best.h264.is_some() { 1 } else { 0 }; - decoding.ability_h265 = if best.h265.is_some() { 1 } else { 0 }; + return VideoCodecState { + score_vpx: SCORE_VPX, + score_h264: best.h264.map_or(0, |c| c.score), + score_h265: best.h265.map_or(0, |c| c.score), + prefer: Self::codec_preference(_id).into(), + ..Default::default() + }; } #[cfg(feature = "mediacodec")] if enable_hwcodec_option() { - decoding.ability_h264 = - if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { - 1 - } else { - 0 - }; - decoding.ability_h265 = - if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { - 1 - } else { - 0 - }; + let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 92 + } else { + 0 + }; + let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 94 + } else { + 0 + }; + return VideoCodecState { + score_vpx: SCORE_VPX, + score_h264, + score_h265, + prefer: Self::codec_preference(_id).into(), + ..Default::default() + }; + } + VideoCodecState { + score_vpx: SCORE_VPX, + ..Default::default() } - decoding } - pub fn new() -> Decoder { - let vp8 = VpxDecoder::new(VpxDecoderConfig { - codec: VpxVideoCodecId::VP8, - num_threads: (num_cpus::get() / 2) as _, - }) - .unwrap(); - let vp9 = VpxDecoder::new(VpxDecoderConfig { - codec: VpxVideoCodecId::VP9, - num_threads: (num_cpus::get() / 2) as _, - }) - .unwrap(); + pub fn new(config: DecoderCfg) -> Decoder { + let vpx = VpxDecoder::new(config.vpx).unwrap(); Decoder { - vp8, - vp9, + vpx, #[cfg(feature = "hwcodec")] hw: if enable_hwcodec_option() { HwDecoder::new_decoders() @@ -291,11 +310,8 @@ impl Decoder { rgb: &mut Vec, ) -> ResultType { match frame { - video_frame::Union::Vp8s(vp8s) => { - Decoder::handle_vpxs_video_frame(&mut self.vp8, vp8s, fmt, rgb) - } video_frame::Union::Vp9s(vp9s) => { - Decoder::handle_vpxs_video_frame(&mut self.vp9, vp9s, fmt, rgb) + Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, fmt, rgb) } #[cfg(feature = "hwcodec")] video_frame::Union::H264s(h264s) => { @@ -333,15 +349,15 @@ impl Decoder { } } - fn handle_vpxs_video_frame( + fn handle_vp9s_video_frame( decoder: &mut VpxDecoder, - vpxs: &EncodedVideoFrames, + vp9s: &EncodedVideoFrames, fmt: (ImageFormat, usize), rgb: &mut Vec, ) -> ResultType { let mut last_frame = Image::new(); - for vpx in vpxs.frames.iter() { - for frame in decoder.decode(&vpx.data)? { + for vp9 in vp9s.frames.iter() { + for frame in decoder.decode(&vp9.data)? { drop(last_frame); last_frame = frame; } @@ -392,15 +408,14 @@ impl Decoder { return Ok(false); } + #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] fn codec_preference(id: &str) -> PreferCodec { let codec = PeerConfig::load(id) .options .get("codec-preference") .map_or("".to_owned(), |c| c.to_owned()); - if codec == "vp8" { - PreferCodec::VP8 - } else if codec == "vp9" { - PreferCodec::VP9 + if codec == "vp9" { + PreferCodec::VPX } else if codec == "h264" { PreferCodec::H264 } else if codec == "h265" { diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index f4de4bf84..2c69774fb 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -7,7 +7,7 @@ use hbb_common::{ anyhow::{anyhow, Context}, bytes::Bytes, config::HwCodecConfig, - log, + lazy_static, log, message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}, ResultType, }; @@ -19,6 +19,11 @@ use hwcodec::{ Quality::{self, *}, RateControl::{self, *}, }; +use std::sync::{Arc, Mutex}; + +lazy_static::lazy_static! { + static ref HW_ENCODER_NAME: Arc>> = Default::default(); +} const CFG_KEY_ENCODER: &str = "bestHwEncoders"; const CFG_KEY_DECODER: &str = "bestHwDecoders"; @@ -44,7 +49,7 @@ impl EncoderApi for HwEncoder { match cfg { EncoderCfg::HW(config) => { let ctx = EncodeContext { - name: config.name.clone(), + name: config.codec_name.clone(), width: config.width as _, height: config.height as _, pixfmt: DEFAULT_PIXFMT, @@ -55,12 +60,12 @@ impl EncoderApi for HwEncoder { quality: DEFAULT_HW_QUALITY, rc: DEFAULT_RC, }; - let format = match Encoder::format_from_name(config.name.clone()) { + let format = match Encoder::format_from_name(config.codec_name.clone()) { Ok(format) => format, Err(_) => { return Err(anyhow!(format!( "failed to get format from name:{}", - config.name + config.codec_name ))) } }; @@ -128,6 +133,10 @@ impl HwEncoder { }) } + pub fn current_name() -> Arc>> { + HW_ENCODER_NAME.clone() + } + pub fn encode(&mut self, bgra: &[u8]) -> ResultType> { match self.pixfmt { AVPixelFormat::AV_PIX_FMT_YUV420P => hw::hw_bgra_to_i420( diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 26b946401..0ad158cca 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -1,5 +1,4 @@ pub use self::vpxcodec::*; -use hbb_common::message_proto::{video_frame, VideoFrame}; cfg_if! { if #[cfg(quartz)] { @@ -93,55 +92,3 @@ pub fn is_cursor_embedded() -> bool { pub fn is_cursor_embedded() -> bool { false } - -#[derive(Debug, Clone, PartialEq, Eq)] -pub enum CodecName { - VP8, - VP9, - H264(String), - H265(String), -} - -#[derive(PartialEq, Debug, Clone)] -pub enum CodecFormat { - VP8, - VP9, - H264, - H265, - Unknown, -} - -impl From<&VideoFrame> for CodecFormat { - fn from(it: &VideoFrame) -> Self { - match it.union { - Some(video_frame::Union::Vp8s(_)) => CodecFormat::VP8, - Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9, - Some(video_frame::Union::H264s(_)) => CodecFormat::H264, - Some(video_frame::Union::H265s(_)) => CodecFormat::H265, - _ => CodecFormat::Unknown, - } - } -} - -impl From<&CodecName> for CodecFormat { - fn from(value: &CodecName) -> Self { - match value { - CodecName::VP8 => Self::VP8, - CodecName::VP9 => Self::VP9, - CodecName::H264(_) => Self::H264, - CodecName::H265(_) => Self::H265, - } - } -} - -impl ToString for CodecFormat { - fn to_string(&self) -> String { - match self { - CodecFormat::VP8 => "VP8".into(), - CodecFormat::VP9 => "VP9".into(), - CodecFormat::H264 => "H264".into(), - CodecFormat::H265 => "H265".into(), - CodecFormat::Unknown => "Unknow".into(), - } - } -} diff --git a/libs/scrap/src/common/record.rs b/libs/scrap/src/common/record.rs index 9de70ae14..9f38f2d6a 100644 --- a/libs/scrap/src/common/record.rs +++ b/libs/scrap/src/common/record.rs @@ -1,4 +1,3 @@ -use crate::CodecFormat; #[cfg(feature = "hwcodec")] use hbb_common::anyhow::anyhow; use hbb_common::{ @@ -22,6 +21,13 @@ use webm::mux::{self, Segment, Track, VideoTrack, Writer}; const MIN_SECS: u64 = 1; +#[derive(Debug, Clone, PartialEq)] +pub enum RecordCodecID { + VP9, + H264, + H265, +} + #[derive(Debug, Clone)] pub struct RecorderContext { pub server: bool, @@ -30,7 +36,7 @@ pub struct RecorderContext { pub filename: String, pub width: usize, pub height: usize, - pub format: CodecFormat, + pub codec_id: RecordCodecID, pub tx: Option>, } @@ -49,9 +55,8 @@ impl RecorderContext { } let file = if self.server { "s" } else { "c" }.to_string() + &self.id.clone() - + &chrono::Local::now().format("_%Y%m%d%H%M%S_").to_string() - + &self.format.to_string() - + if self.format == CodecFormat::VP9 || self.format == CodecFormat::VP8 { + + &chrono::Local::now().format("_%Y%m%d%H%M%S").to_string() + + if self.codec_id == RecordCodecID::VP9 { ".webm" } else { ".mp4" @@ -102,8 +107,8 @@ impl DerefMut for Recorder { impl Recorder { pub fn new(mut ctx: RecorderContext) -> ResultType { ctx.set_filename()?; - let recorder = match ctx.format { - CodecFormat::VP8 | CodecFormat::VP9 => Recorder { + let recorder = match ctx.codec_id { + RecordCodecID::VP9 => Recorder { inner: Box::new(WebmRecorder::new(ctx.clone())?), ctx, }, @@ -121,8 +126,8 @@ impl Recorder { fn change(&mut self, mut ctx: RecorderContext) -> ResultType<()> { ctx.set_filename()?; - self.inner = match ctx.format { - CodecFormat::VP8 | CodecFormat::VP9 => Box::new(WebmRecorder::new(ctx.clone())?), + self.inner = match ctx.codec_id { + RecordCodecID::VP9 => Box::new(WebmRecorder::new(ctx.clone())?), #[cfg(feature = "hwcodec")] _ => Box::new(HwRecorder::new(ctx.clone())?), #[cfg(not(feature = "hwcodec"))] @@ -143,19 +148,10 @@ impl Recorder { pub fn write_frame(&mut self, frame: &video_frame::Union) -> ResultType<()> { match frame { - video_frame::Union::Vp8s(vp8s) => { - if self.ctx.format != CodecFormat::VP8 { - self.change(RecorderContext { - format: CodecFormat::VP8, - ..self.ctx.clone() - })?; - } - vp8s.frames.iter().map(|f| self.write_video(f)).count(); - } video_frame::Union::Vp9s(vp9s) => { - if self.ctx.format != CodecFormat::VP9 { + if self.ctx.codec_id != RecordCodecID::VP9 { self.change(RecorderContext { - format: CodecFormat::VP9, + codec_id: RecordCodecID::VP9, ..self.ctx.clone() })?; } @@ -163,25 +159,25 @@ impl Recorder { } #[cfg(feature = "hwcodec")] video_frame::Union::H264s(h264s) => { - if self.ctx.format != CodecFormat::H264 { + if self.ctx.codec_id != RecordCodecID::H264 { self.change(RecorderContext { - format: CodecFormat::H264, + codec_id: RecordCodecID::H264, ..self.ctx.clone() })?; } - if self.ctx.format == CodecFormat::H264 { + if self.ctx.codec_id == RecordCodecID::H264 { h264s.frames.iter().map(|f| self.write_video(f)).count(); } } #[cfg(feature = "hwcodec")] video_frame::Union::H265s(h265s) => { - if self.ctx.format != CodecFormat::H265 { + if self.ctx.codec_id != RecordCodecID::H265 { self.change(RecorderContext { - format: CodecFormat::H265, + codec_id: RecordCodecID::H265, ..self.ctx.clone() })?; } - if self.ctx.format == CodecFormat::H265 { + if self.ctx.codec_id == RecordCodecID::H265 { h265s.frames.iter().map(|f| self.write_video(f)).count(); } } @@ -225,11 +221,7 @@ impl RecorderApi for WebmRecorder { ctx.width as _, ctx.height as _, None, - if ctx.format == CodecFormat::VP9 { - mux::VideoCodecId::VP9 - } else { - mux::VideoCodecId::VP8 - }, + mux::VideoCodecId::VP9, ); Ok(WebmRecorder { vt, @@ -287,7 +279,7 @@ impl RecorderApi for HwRecorder { filename: ctx.filename.clone(), width: ctx.width, height: ctx.height, - is265: ctx.format == CodecFormat::H265, + is265: ctx.codec_id == RecordCodecID::H265, framerate: crate::hwcodec::DEFAULT_TIME_BASE[1] as _, }) .map_err(|_| anyhow!("Failed to create hardware muxer"))?; diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index 4820ea171..3df9c0461 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -30,7 +30,6 @@ pub struct VpxEncoder { ctx: vpx_codec_ctx_t, width: usize, height: usize, - id: VpxVideoCodecId, } pub struct VpxDecoder { @@ -98,10 +97,15 @@ impl EncoderApi for VpxEncoder { { match cfg { crate::codec::EncoderCfg::VPX(config) => { - let i = match config.codec { - VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), - VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), - }; + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_cx()); + } let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); @@ -183,17 +187,12 @@ impl EncoderApi for VpxEncoder { VP9E_SET_TILE_COLUMNS as _, 4 as c_int )); - } else if config.codec == VpxVideoCodecId::VP8 { - // https://github.com/webmproject/libvpx/blob/972149cafeb71d6f08df89e91a0130d6a38c4b15/vpx/vp8cx.h#L172 - // https://groups.google.com/a/webmproject.org/g/webm-discuss/c/DJhSrmfQ61M - call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 12,)); } Ok(Self { ctx, width: config.width as _, height: config.height as _, - id: config.codec, }) } _ => Err(anyhow!("encoder type mismatch")), @@ -214,7 +213,7 @@ impl EncoderApi for VpxEncoder { // to-do: flush periodically, e.g. 1 second if frames.len() > 0 { - Ok(VpxEncoder::create_msg(self.id, frames)) + Ok(VpxEncoder::create_msg(frames)) } else { Err(anyhow!("no valid frame")) } @@ -281,17 +280,13 @@ impl VpxEncoder { } #[inline] - pub fn create_msg(codec_id: VpxVideoCodecId, frames: Vec) -> Message { + fn create_msg(vp9s: Vec) -> Message { let mut msg_out = Message::new(); let mut vf = VideoFrame::new(); - let vpxs = EncodedVideoFrames { - frames: frames.into(), + vf.set_vp9s(EncodedVideoFrames { + frames: vp9s.into(), ..Default::default() - }; - match codec_id { - VpxVideoCodecId::VP8 => vf.set_vp8s(vpxs), - VpxVideoCodecId::VP9 => vf.set_vp9s(vpxs), - } + }); msg_out.set_video_frame(vf); msg_out } @@ -387,10 +382,15 @@ impl VpxDecoder { pub fn new(config: VpxDecoderConfig) -> Result { // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can // cause UB if uninitialized. - let i = match config.codec { - VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), - VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), - }; + let i; + if cfg!(feature = "VP8") { + i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), + }; + } else { + i = call_vpx_ptr!(vpx_codec_vp9_dx()); + } let mut ctx = Default::default(); let cfg = vpx_codec_dec_cfg_t { threads: if config.num_threads == 0 { diff --git a/src/client.rs b/src/client.rs index ae93ccfcf..2f745b70c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -44,9 +44,9 @@ use hbb_common::{ }; pub use helper::*; use scrap::{ - codec::Decoder, + codec::{Decoder, DecoderCfg}, record::{Recorder, RecorderContext}, - ImageFormat, + ImageFormat, VpxDecoderConfig, VpxVideoCodecId, }; use crate::{ @@ -917,7 +917,12 @@ impl VideoHandler { /// Create a new video handler. pub fn new() -> Self { VideoHandler { - decoder: Decoder::new(), + decoder: Decoder::new(DecoderCfg { + vpx: VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: (num_cpus::get() / 2) as _, + }, + }), rgb: Default::default(), recorder: Default::default(), record: false, @@ -949,7 +954,12 @@ impl VideoHandler { /// Reset the decoder. pub fn reset(&mut self) { - self.decoder = Decoder::new(); + self.decoder = Decoder::new(DecoderCfg { + vpx: VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: 1, + }, + }); } /// Start or stop screen record. @@ -963,7 +973,7 @@ impl VideoHandler { filename: "".to_owned(), width: w as _, height: h as _, - format: scrap::CodecFormat::VP9, + codec_id: scrap::record::RecordCodecID::VP9, tx: None, }) .map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r)))); @@ -989,7 +999,7 @@ pub struct LoginConfigHandler { pub conn_id: i32, features: Option, session_id: u64, - pub supported_encoding: SupportedEncoding, + pub supported_encoding: Option<(bool, bool)>, pub restarting_remote_device: bool, pub force_relay: bool, pub direct: Option, @@ -1037,7 +1047,7 @@ impl LoginConfigHandler { self.remember = !config.password.is_empty(); self.config = config; self.session_id = rand::random(); - self.supported_encoding = Default::default(); + self.supported_encoding = None; self.restarting_remote_device = false; self.force_relay = !self.get_option("force-always-relay").is_empty() || force_relay; self.direct = None; @@ -1321,8 +1331,8 @@ impl LoginConfigHandler { msg.disable_clipboard = BoolOption::Yes.into(); n += 1; } - msg.supported_decoding = - hbb_common::protobuf::MessageField::some(Decoder::supported_decodings(Some(&self.id))); + let state = Decoder::video_codec_state(&self.id); + msg.video_codec_state = hbb_common::protobuf::MessageField::some(state); n += 1; if n > 0 { @@ -1555,7 +1565,10 @@ impl LoginConfigHandler { self.conn_id = pi.conn_id; // no matter if change, for update file time self.save_config(config); - self.supported_encoding = pi.encoding.clone().unwrap_or_default(); + #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] + { + self.supported_encoding = Some((pi.encoding.h264, pi.encoding.h265)); + } } pub fn get_remote_dir(&self) -> String { @@ -1613,10 +1626,10 @@ impl LoginConfigHandler { } pub fn change_prefer_codec(&self) -> Message { - let decoding = scrap::codec::Decoder::supported_decodings(Some(&self.id)); + let state = scrap::codec::Decoder::video_codec_state(&self.id); let mut misc = Misc::new(); misc.set_option(OptionMessage { - supported_decoding: hbb_common::protobuf::MessageField::some(decoding), + video_codec_state: hbb_common::protobuf::MessageField::some(state), ..Default::default() }); let mut msg_out = Message::new(); diff --git a/src/client/helper.rs b/src/client/helper.rs index 61844d908..a9696a8e8 100644 --- a/src/client/helper.rs +++ b/src/client/helper.rs @@ -1,8 +1,37 @@ use hbb_common::{ get_time, - message_proto::{Message, VoiceCallRequest, VoiceCallResponse}, + message_proto::{video_frame, Message, VideoFrame, VoiceCallRequest, VoiceCallResponse}, }; -use scrap::CodecFormat; + +#[derive(PartialEq, Debug, Clone)] +pub enum CodecFormat { + VP9, + H264, + H265, + Unknown, +} + +impl From<&VideoFrame> for CodecFormat { + fn from(it: &VideoFrame) -> Self { + match it.union { + Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9, + Some(video_frame::Union::H264s(_)) => CodecFormat::H264, + Some(video_frame::Union::H265s(_)) => CodecFormat::H265, + _ => CodecFormat::Unknown, + } + } +} + +impl ToString for CodecFormat { + fn to_string(&self) -> String { + match self { + CodecFormat::VP9 => "VP9".into(), + CodecFormat::H264 => "H264".into(), + CodecFormat::H265 => "H265".into(), + CodecFormat::Unknown => "Unknow".into(), + } + } +} #[derive(Debug, Default)] pub struct QualityStatus { diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index 0f662e015..b0bddc82e 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -29,10 +29,10 @@ use hbb_common::tokio::{ time::{self, Duration, Instant, Interval}, }; use hbb_common::{allow_err, fs, get_time, log, message_proto::*, Stream}; -use scrap::CodecFormat; use crate::client::{ - new_voice_call_request, Client, MediaData, MediaSender, QualityStatus, MILLI1, SEC30, + new_voice_call_request, Client, CodecFormat, MediaData, MediaSender, QualityStatus, MILLI1, + SEC30, }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::{self, update_clipboard}; @@ -817,10 +817,11 @@ impl Remote { } fn contains_key_frame(vf: &VideoFrame) -> bool { - use video_frame::Union::*; match &vf.union { Some(vf) => match vf { - Vp8s(f) | Vp9s(f) | H264s(f) | H265s(f) => f.frames.iter().any(|e| e.key), + video_frame::Union::Vp9s(f) => f.frames.iter().any(|e| e.key), + video_frame::Union::H264s(f) => f.frames.iter().any(|e| e.key), + video_frame::Union::H265s(f) => f.frames.iter().any(|e| e.key), _ => false, }, None => false, diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 1d965ebc5..aee486b94 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -969,13 +969,6 @@ pub fn main_has_hwcodec() -> SyncReturn { SyncReturn(has_hwcodec()) } -pub fn main_supported_hwdecodings() -> SyncReturn { - let decoding = supported_hwdecodings(); - let msg = HashMap::from([("h264", decoding.0), ("h265", decoding.1)]); - - SyncReturn(serde_json::ser::to_string(&msg).unwrap_or("".to_owned())) -} - pub fn main_is_root() -> bool { is_root() } @@ -1061,10 +1054,10 @@ pub fn session_send_note(id: String, note: String) { } } -pub fn session_alternative_codecs(id: String) -> String { +pub fn session_supported_hwcodec(id: String) -> String { if let Some(session) = SESSIONS.read().unwrap().get(&id) { - let (vp8, h264, h265) = session.alternative_codecs(); - let msg = HashMap::from([("vp8", vp8), ("h264", h264), ("h265", h265)]); + let (h264, h265) = session.supported_hwcodec(); + let msg = HashMap::from([("h264", h264), ("h265", h265)]); serde_json::ser::to_string(&msg).unwrap_or("".to_owned()) } else { String::new() diff --git a/src/server/connection.rs b/src/server/connection.rs index d1ae4d6a3..23e166fcf 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -536,7 +536,7 @@ impl Connection { let _ = privacy_mode::turn_off_privacy(0); } video_service::notify_video_frame_fetched(id, None); - scrap::codec::Encoder::update(id, scrap::codec::EncodingUpdate::Remove); + scrap::codec::Encoder::update_video_encoder(id, scrap::codec::EncoderUpdate::Remove); video_service::VIDEO_QOS.lock().unwrap().reset(); if conn.authorized { password::update_temporary_password(); @@ -862,7 +862,16 @@ impl Connection { } } - pi.encoding = Some(scrap::codec::Encoder::supported_encoding()).into(); + #[cfg(feature = "hwcodec")] + { + let (h264, h265) = scrap::codec::Encoder::supported_encoding(); + pi.encoding = Some(SupportedEncoding { + h264, + h265, + ..Default::default() + }) + .into(); + } if self.port_forward_socket.is_some() { let mut msg_out = Message::new(); @@ -1138,21 +1147,21 @@ impl Connection { self.lr = lr.clone(); if let Some(o) = lr.option.as_ref() { self.options_in_login = Some(o.clone()); - if let Some(q) = o.supported_decoding.clone().take() { - scrap::codec::Encoder::update( + if let Some(q) = o.video_codec_state.clone().take() { + scrap::codec::Encoder::update_video_encoder( self.inner.id(), - scrap::codec::EncodingUpdate::New(q), + scrap::codec::EncoderUpdate::State(q), ); } else { - scrap::codec::Encoder::update( + scrap::codec::Encoder::update_video_encoder( self.inner.id(), - scrap::codec::EncodingUpdate::NewOnlyVP9, + scrap::codec::EncoderUpdate::DisableHwIfNotExist, ); } } else { - scrap::codec::Encoder::update( + scrap::codec::Encoder::update_video_encoder( self.inner.id(), - scrap::codec::EncodingUpdate::NewOnlyVP9, + scrap::codec::EncoderUpdate::DisableHwIfNotExist, ); } self.video_ack_required = lr.video_ack_required; @@ -1775,8 +1784,11 @@ impl Connection { .unwrap() .update_user_fps(o.custom_fps as _); } - if let Some(q) = o.supported_decoding.clone().take() { - scrap::codec::Encoder::update(self.inner.id(), scrap::codec::EncodingUpdate::New(q)); + if let Some(q) = o.video_codec_state.clone().take() { + scrap::codec::Encoder::update_video_encoder( + self.inner.id(), + scrap::codec::EncoderUpdate::State(q), + ); } if let Ok(q) = o.lock_after_session_end.enum_value() { if q != BoolOption::NotSet { diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 691ca4abe..205d0584c 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -31,7 +31,7 @@ use scrap::{ codec::{Encoder, EncoderCfg, HwEncoderConfig}, record::{Recorder, RecorderContext}, vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, - CodecName, Display, TraitCapturer, + Display, TraitCapturer, }; #[cfg(windows)] use std::sync::Once; @@ -468,29 +468,21 @@ fn run(sp: GenericService) -> ResultType<()> { drop(video_qos); log::info!("init bitrate={}, abr enabled:{}", bitrate, abr); - let encoder_cfg = match Encoder::negotiated_codec() { - scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => { - EncoderCfg::HW(HwEncoderConfig { - name, - width: c.width, - height: c.height, - bitrate: bitrate as _, - }) - } - name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => { - EncoderCfg::VPX(VpxEncoderConfig { - width: c.width as _, - height: c.height as _, - timebase: [1, 1000], // Output timestamp precision - bitrate, - codec: if name == scrap::CodecName::VP8 { - VpxVideoCodecId::VP8 - } else { - VpxVideoCodecId::VP9 - }, - num_threads: (num_cpus::get() / 2) as _, - }) - } + let encoder_cfg = match Encoder::current_hw_encoder_name() { + Some(codec_name) => EncoderCfg::HW(HwEncoderConfig { + codec_name, + width: c.width, + height: c.height, + bitrate: bitrate as _, + }), + None => EncoderCfg::VPX(VpxEncoderConfig { + width: c.width as _, + height: c.height as _, + timebase: [1, 1000], // Output timestamp precision + bitrate, + codec: VpxVideoCodecId::VP9, + num_threads: (num_cpus::get() / 2) as _, + }), }; let mut encoder; @@ -534,7 +526,7 @@ fn run(sp: GenericService) -> ResultType<()> { let mut try_gdi = 1; #[cfg(windows)] log::info!("gdi: {}", c.is_gdi()); - let codec_name = Encoder::negotiated_codec(); + let codec_name = Encoder::current_hw_encoder_name(); let recorder = get_recorder(c.width, c.height, &codec_name); #[cfg(windows)] start_uac_elevation_check(); @@ -565,7 +557,7 @@ fn run(sp: GenericService) -> ResultType<()> { *SWITCH.lock().unwrap() = true; bail!("SWITCH"); } - if codec_name != Encoder::negotiated_codec() { + if codec_name != Encoder::current_hw_encoder_name() { bail!("SWITCH"); } #[cfg(windows)] @@ -611,14 +603,8 @@ fn run(sp: GenericService) -> ResultType<()> { let time = now - start; let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; match frame { - scrap::Frame::VP8(data) => { - let send_conn_ids = - handle_one_frame_encoded(VpxVideoCodecId::VP8, &sp, data, ms)?; - frame_controller.set_send(now, send_conn_ids); - } scrap::Frame::VP9(data) => { - let send_conn_ids = - handle_one_frame_encoded(VpxVideoCodecId::VP9, &sp, data, ms)?; + let send_conn_ids = handle_one_frame_encoded(&sp, data, ms)?; frame_controller.set_send(now, send_conn_ids); } scrap::Frame::RAW(data) => { @@ -731,11 +717,12 @@ fn run(sp: GenericService) -> ResultType<()> { fn get_recorder( width: usize, height: usize, - codec_name: &CodecName, + codec_name: &Option, ) -> Arc>> { #[cfg(not(target_os = "ios"))] let recorder = if !Config::get_option("allow-auto-record-incoming").is_empty() { use crate::hbbs_http::record_upload; + use scrap::record::RecordCodecID::*; let tx = if record_upload::is_enable() { let (tx, rx) = std::sync::mpsc::channel(); @@ -744,6 +731,16 @@ fn get_recorder( } else { None }; + let codec_id = match codec_name { + Some(name) => { + if name.contains("264") { + H264 + } else { + H265 + } + } + None => VP9, + }; Recorder::new(RecorderContext { server: true, id: Config::get_id(), @@ -751,7 +748,7 @@ fn get_recorder( filename: "".to_owned(), width, height, - format: codec_name.into(), + codec_id, tx, }) .map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r)))) @@ -778,6 +775,19 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu Ok(()) } +#[inline] +#[cfg(any(target_os = "android", target_os = "ios"))] +fn create_msg(vp9s: Vec) -> Message { + let mut msg_out = Message::new(); + let mut vf = VideoFrame::new(); + vf.set_vp9s(EncodedVideoFrames { + frames: vp9s.into(), + ..Default::default() + }); + msg_out.set_video_frame(vf); + msg_out +} + #[inline] fn handle_one_frame( sp: &GenericService, @@ -810,7 +820,6 @@ fn handle_one_frame( #[inline] #[cfg(any(target_os = "android", target_os = "ios"))] pub fn handle_one_frame_encoded( - codec: VpxVideoCodecId, sp: &GenericService, frame: &[u8], ms: i64, @@ -822,13 +831,13 @@ pub fn handle_one_frame_encoded( } Ok(()) })?; - let vpx_frame = EncodedVideoFrame { + let vp9_frame = EncodedVideoFrame { data: frame.to_vec().into(), key: true, pts: ms, ..Default::default() }; - let send_conn_ids = sp.send_video_frame(scrap::VpxEncoder::create_msg(codec, vec![vpx_frame])); + let send_conn_ids = sp.send_video_frame(create_msg(vec![vp9_frame])); Ok(send_conn_ids) } diff --git a/src/ui/header.tis b/src/ui/header.tis index af4f1e349..257ba417e 100644 --- a/src/ui/header.tis +++ b/src/ui/header.tis @@ -161,8 +161,8 @@ class Header: Reactor.Component { } function renderDisplayPop() { - var codecs = handler.alternative_codecs(); - var show_codec = codecs[0] || codecs[1] || codecs[2]; + var codecs = handler.supported_hwcodec(); + var show_codec = handler.has_hwcodec() && (codecs[0] || codecs[1]); var cursor_embedded = false; if ((pi.displays || []).length > 0) { @@ -186,10 +186,9 @@ class Header: Reactor.Component { {show_codec ?
  • {svg_checkmark}Auto
  • - {codecs[0] ?
  • {svg_checkmark}VP8
  • : ""}
  • {svg_checkmark}VP9
  • - {codecs[1] ?
  • {svg_checkmark}H264
  • : ""} - {codecs[2] ?
  • {svg_checkmark}H265
  • : ""} + {codecs[0] ?
  • {svg_checkmark}H264
  • : ""} + {codecs[1] ?
  • {svg_checkmark}H265
  • : ""}
    : ""}
    {!cursor_embedded &&
  • {svg_checkmark}{translate('Show remote cursor')}
  • } diff --git a/src/ui/remote.rs b/src/ui/remote.rs index fc878cf1f..68decf955 100644 --- a/src/ui/remote.rs +++ b/src/ui/remote.rs @@ -20,6 +20,7 @@ use hbb_common::{ use crate::{ client::*, + ui_interface::has_hwcodec, ui_session_interface::{InvokeUiSession, Session}, }; @@ -196,14 +197,7 @@ impl InvokeUiSession for SciterHandler { self.call("confirmDeleteFiles", &make_args!(id, i, name)); } - fn override_file_confirm( - &self, - id: i32, - file_num: i32, - to: String, - is_upload: bool, - is_identical: bool, - ) { + fn override_file_confirm(&self, id: i32, file_num: i32, to: String, is_upload: bool, is_identical: bool) { self.call( "overrideFileConfirm", &make_args!(id, file_num, to, is_upload, is_identical), @@ -457,7 +451,8 @@ impl sciter::EventHandler for SciterSession { fn set_write_override(i32, i32, bool, bool, bool); fn get_keyboard_mode(); fn save_keyboard_mode(String); - fn alternative_codecs(); + fn has_hwcodec(); + fn supported_hwcodec(); fn change_prefer_codec(); fn restart_remote_device(); fn request_voice_call(); @@ -509,6 +504,10 @@ impl SciterSession { v } + fn has_hwcodec(&self) -> bool { + has_hwcodec() + } + pub fn t(&self, name: String) -> String { crate::client::translate(name) } @@ -517,10 +516,9 @@ impl SciterSession { super::get_icon() } - fn alternative_codecs(&self) -> Value { - let (vp8, h264, h265) = self.0.alternative_codecs(); + fn supported_hwcodec(&self) -> Value { + let (h264, h265) = self.0.supported_hwcodec(); let mut v = Value::array(0); - v.push(vp8); v.push(h264); v.push(h265); v diff --git a/src/ui_interface.rs b/src/ui_interface.rs index 26d470fe0..11357be4a 100644 --- a/src/ui_interface.rs +++ b/src/ui_interface.rs @@ -752,13 +752,6 @@ pub fn has_hwcodec() -> bool { return true; } -#[cfg(feature = "flutter")] -#[inline] -pub fn supported_hwdecodings() -> (bool, bool) { - let decoding = scrap::codec::Decoder::supported_decodings(None); - (decoding.ability_h264 > 0, decoding.ability_h265 > 0) -} - #[cfg(not(any(target_os = "android", target_os = "ios")))] #[inline] pub fn is_root() -> bool { diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 3dee89a6e..f89be4879 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -216,16 +216,24 @@ impl Session { true } - pub fn alternative_codecs(&self) -> (bool, bool, bool) { - let decoder = scrap::codec::Decoder::supported_decodings(None); - let mut vp8 = decoder.ability_vp8 > 0; - let mut h264 = decoder.ability_h264 > 0; - let mut h265 = decoder.ability_h265 > 0; - let enc = &self.lc.read().unwrap().supported_encoding; - vp8 = vp8 && enc.vp8; - h264 = h264 && enc.h264; - h265 = h265 && enc.h265; - (vp8, h264, h265) + pub fn supported_hwcodec(&self) -> (bool, bool) { + #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] + { + let decoder = scrap::codec::Decoder::video_codec_state(&self.id); + let mut h264 = decoder.score_h264 > 0; + let mut h265 = decoder.score_h265 > 0; + let (encoding_264, encoding_265) = self + .lc + .read() + .unwrap() + .supported_encoding + .unwrap_or_default(); + h264 = h264 && encoding_264; + h265 = h265 && encoding_265; + return (h264, h265); + } + #[allow(unreachable_code)] + (false, false) } pub fn change_prefer_codec(&self) {