From 78748271ac329a63fea837d772a7f7da4f462f89 Mon Sep 17 00:00:00 2001 From: 21pages Date: Fri, 31 Mar 2023 16:10:52 +0800 Subject: [PATCH] vp8 Signed-off-by: 21pages --- .../desktop/pages/desktop_setting_page.dart | 42 ++- .../lib/desktop/widgets/remote_toolbar.dart | 26 +- flutter/lib/mobile/pages/remote_page.dart | 5 +- libs/hbb_common/protos/message.proto | 16 +- libs/hbb_common/src/config.rs | 5 +- libs/scrap/examples/benchmark.rs | 46 ++- libs/scrap/src/common/android.rs | 1 + libs/scrap/src/common/codec.rs | 319 +++++++++--------- libs/scrap/src/common/hwcodec.rs | 17 +- libs/scrap/src/common/mod.rs | 53 +++ libs/scrap/src/common/record.rs | 58 ++-- libs/scrap/src/common/vpxcodec.rs | 46 +-- src/client.rs | 37 +- src/client/helper.rs | 33 +- src/client/io_loop.rs | 9 +- src/flutter_ffi.rs | 13 +- src/server/connection.rs | 34 +- src/server/video_service.rs | 85 +++-- src/ui/header.tis | 9 +- src/ui/remote.rs | 22 +- src/ui_interface.rs | 7 + src/ui_session_interface.rs | 28 +- 22 files changed, 470 insertions(+), 441 deletions(-) diff --git a/flutter/lib/desktop/pages/desktop_setting_page.dart b/flutter/lib/desktop/pages/desktop_setting_page.dart index 66ef83d31..74d51407c 100644 --- a/flutter/lib/desktop/pages/desktop_setting_page.dart +++ b/flutter/lib/desktop/pages/desktop_setting_page.dart @@ -1258,9 +1258,6 @@ class _DisplayState extends State<_Display> { } Widget codec(BuildContext context) { - if (!bind.mainHasHwcodec()) { - return Offstage(); - } final key = 'codec-preference'; onChanged(String value) async { await bind.mainSetUserDefaultOption(key: key, value: value); @@ -1268,28 +1265,45 @@ class _DisplayState extends State<_Display> { } final groupValue = bind.mainGetUserDefaultOption(key: key); - + var hwRadios = []; + try { + final Map codecsJson = jsonDecode(bind.mainSupportedHwdecodings()); + final h264 = codecsJson['h264'] ?? false; + final h265 = codecsJson['h265'] ?? false; + if (h264) { + hwRadios.add(_Radio(context, + value: 'h264', + groupValue: groupValue, + label: 'H264', + onChanged: onChanged)); + } + if (h265) { + hwRadios.add(_Radio(context, + value: 'h265', + groupValue: groupValue, + label: 'H265', + onChanged: onChanged)); + } + } catch (e) { + debugPrint("failed to parse supported hwdecodings, err=$e"); + } return _Card(title: 'Default Codec', children: [ _Radio(context, value: 'auto', groupValue: groupValue, label: 'Auto', onChanged: onChanged), + _Radio(context, + value: 'vp8', + groupValue: groupValue, + label: 'VP8', + onChanged: onChanged), _Radio(context, value: 'vp9', groupValue: groupValue, label: 'VP9', onChanged: onChanged), - _Radio(context, - value: 'h264', - groupValue: groupValue, - label: 'H264', - onChanged: onChanged), - _Radio(context, - value: 'h265', - groupValue: groupValue, - label: 'H265', - onChanged: onChanged), + ...hwRadios, ]); } diff --git a/flutter/lib/desktop/widgets/remote_toolbar.dart b/flutter/lib/desktop/widgets/remote_toolbar.dart index 0ef8674ef..cb4b6d644 100644 --- a/flutter/lib/desktop/widgets/remote_toolbar.dart +++ b/flutter/lib/desktop/widgets/remote_toolbar.dart @@ -1349,29 +1349,30 @@ class _DisplayMenuState extends State<_DisplayMenu> { codec() { return futureBuilder(future: () async { - final supportedHwcodec = - await bind.sessionSupportedHwcodec(id: widget.id); + final alternativeCodecs = + await bind.sessionAlternativeCodecs(id: widget.id); final codecPreference = await bind.sessionGetOption(id: widget.id, arg: 'codec-preference') ?? ''; return { - 'supportedHwcodec': supportedHwcodec, + 'alternativeCodecs': alternativeCodecs, 'codecPreference': codecPreference }; }(), hasData: (data) { final List codecs = []; try { - final Map codecsJson = jsonDecode(data['supportedHwcodec']); + final Map codecsJson = jsonDecode(data['alternativeCodecs']); + final vp8 = codecsJson['vp8'] ?? false; final h264 = codecsJson['h264'] ?? false; final h265 = codecsJson['h265'] ?? false; + codecs.add(vp8); codecs.add(h264); codecs.add(h265); } catch (e) { debugPrint("Show Codec Preference err=$e"); } - final visible = bind.mainHasHwcodec() && - codecs.length == 2 && - (codecs[0] || codecs[1]); + final visible = + codecs.length == 3 && (codecs[0] || codecs[1] || codecs[2]); if (!visible) return Offstage(); final groupValue = data['codecPreference'] as String; onChanged(String? value) async { @@ -1392,6 +1393,13 @@ class _DisplayMenuState extends State<_DisplayMenu> { onChanged: onChanged, ffi: widget.ffi, ), + _RadioMenuButton( + child: Text(translate('VP8')), + value: 'vp8', + groupValue: groupValue, + onChanged: codecs[0] ? onChanged : null, + ffi: widget.ffi, + ), _RadioMenuButton( child: Text(translate('VP9')), value: 'vp9', @@ -1403,14 +1411,14 @@ class _DisplayMenuState extends State<_DisplayMenu> { child: Text(translate('H264')), value: 'h264', groupValue: groupValue, - onChanged: codecs[0] ? onChanged : null, + onChanged: codecs[1] ? onChanged : null, ffi: widget.ffi, ), _RadioMenuButton( child: Text(translate('H265')), value: 'h265', groupValue: groupValue, - onChanged: codecs[1] ? onChanged : null, + onChanged: codecs[2] ? onChanged : null, ffi: widget.ffi, ), ]); diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index 083cdcd1c..74a327c27 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -973,9 +973,11 @@ void showOptions( if (hasHwcodec) { try { final Map codecsJson = - jsonDecode(await bind.sessionSupportedHwcodec(id: id)); + jsonDecode(await bind.sessionAlternativeCodecs(id: id)); + final vp8 = codecsJson['vp8'] ?? false; final h264 = codecsJson['h264'] ?? false; final h265 = codecsJson['h265'] ?? false; + codecs.add(vp8); codecs.add(h264); codecs.add(h265); } catch (e) { @@ -1044,6 +1046,7 @@ void showOptions( if (hasHwcodec && codecs.length == 2 && (codecs[0] || codecs[1])) { radios.addAll([ getRadio(translate('Auto'), 'auto', codec, setCodec), + getRadio('VP8', 'vp8', codec, setCodec), getRadio('VP9', 'vp9', codec, setCodec), ]); if (codecs[0]) { diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 6295c160b..321747e3a 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -24,6 +24,7 @@ message VideoFrame { YUV yuv = 8; EncodedVideoFrames h264s = 10; EncodedVideoFrames h265s = 11; + EncodedVideoFrames vp8s = 12; } } @@ -76,6 +77,7 @@ message Features { message SupportedEncoding { bool h264 = 1; bool h265 = 2; + bool vp8 = 3; } message PeerInfo { @@ -457,18 +459,20 @@ enum ImageQuality { Best = 4; } -message VideoCodecState { +message SupportedDecoding { enum PreferCodec { Auto = 0; - VPX = 1; + VP9 = 1; H264 = 2; H265 = 3; + VP8 = 4; } - int32 score_vpx = 1; - int32 score_h264 = 2; - int32 score_h265 = 3; + int32 ability_vp9 = 1; + int32 ability_h264 = 2; + int32 ability_h265 = 3; PreferCodec prefer = 4; + int32 ability_vp8 = 5; } message OptionMessage { @@ -486,7 +490,7 @@ message OptionMessage { BoolOption disable_audio = 7; BoolOption disable_clipboard = 8; BoolOption enable_file_transfer = 9; - VideoCodecState video_codec_state = 10; + SupportedDecoding supported_decoding = 10; int32 custom_fps = 11; BoolOption disable_keyboard = 12; } diff --git a/libs/hbb_common/src/config.rs b/libs/hbb_common/src/config.rs index 6a823c7b7..960074a8f 100644 --- a/libs/hbb_common/src/config.rs +++ b/libs/hbb_common/src/config.rs @@ -917,7 +917,8 @@ impl PeerConfig { store = store || store2; for opt in ["rdp_password", "os-password"] { if let Some(v) = config.options.get_mut(opt) { - let (encrypted, _, store2) = decrypt_str_or_original(v, PASSWORD_ENC_VERSION); + let (encrypted, _, store2) = + decrypt_str_or_original(v, PASSWORD_ENC_VERSION); *v = encrypted; store = store || store2; } @@ -1356,7 +1357,7 @@ impl UserDefaultConfig { "view_style" => self.get_string(key, "original", vec!["adaptive"]), "scroll_style" => self.get_string(key, "scrollauto", vec!["scrollbar"]), "image_quality" => self.get_string(key, "balanced", vec!["best", "low", "custom"]), - "codec-preference" => self.get_string(key, "auto", vec!["vp9", "h264", "h265"]), + "codec-preference" => self.get_string(key, "auto", vec!["vp8", "vp9", "h264", "h265"]), "custom_image_quality" => self.get_double_string(key, 50.0, 10.0, 100.0), "custom-fps" => self.get_double_string(key, 30.0, 10.0, 120.0), _ => self diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index 003830f95..ba8dec9f2 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -3,7 +3,8 @@ use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV}; use scrap::{ codec::{EncoderApi, EncoderCfg}, Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, - VpxVideoCodecId, STRIDE_ALIGN, + VpxVideoCodecId::{self, *}, + STRIDE_ALIGN, }; use std::{io::Write, time::Instant}; @@ -49,7 +50,7 @@ fn main() { "benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}", width, height, bitrate_k, args.flag_hw_pixfmt ); - test_vp9(&yuvs, width, height, bitrate_k, yuv_count); + [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count)); #[cfg(feature = "hwcodec")] { use hwcodec::AVPixelFormat; @@ -57,7 +58,7 @@ fn main() { Pixfmt::I420 => AVPixelFormat::AV_PIX_FMT_YUV420P, Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12, }; - let yuvs = hw::vp9_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt); + let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt); hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt); } } @@ -87,13 +88,20 @@ fn capture_yuv(yuv_count: usize) -> (Vec>, usize, usize) { } } -fn test_vp9(yuvs: &Vec>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) { +fn test_vpx( + codec_id: VpxVideoCodecId, + yuvs: &Vec>, + width: usize, + height: usize, + bitrate_k: usize, + yuv_count: usize, +) { let config = EncoderCfg::VPX(VpxEncoderConfig { width: width as _, height: height as _, timebase: [1, 1000], bitrate: bitrate_k as _, - codec: VpxVideoCodecId::VP9, + codec: codec_id, num_threads: (num_cpus::get() / 2) as _, }); let mut encoder = VpxEncoder::new(config).unwrap(); @@ -104,35 +112,43 @@ fn test_vp9(yuvs: &Vec>, width: usize, height: usize, bitrate_k: usize, .unwrap(); let _ = encoder.flush().unwrap(); } - println!("vp9 encode: {:?}", start.elapsed() / yuv_count as _); + println!( + "{:?} encode: {:?}", + codec_id, + start.elapsed() / yuv_count as _ + ); // prepare data separately - let mut vp9s = vec![]; + let mut vpxs = vec![]; let start = Instant::now(); for yuv in yuvs { for ref frame in encoder .encode(start.elapsed().as_millis() as _, yuv, STRIDE_ALIGN) .unwrap() { - vp9s.push(frame.data.to_vec()); + vpxs.push(frame.data.to_vec()); } for ref frame in encoder.flush().unwrap() { - vp9s.push(frame.data.to_vec()); + vpxs.push(frame.data.to_vec()); } } - assert_eq!(vp9s.len(), yuv_count); + assert_eq!(vpxs.len(), yuv_count); let mut decoder = VpxDecoder::new(VpxDecoderConfig { - codec: VpxVideoCodecId::VP9, + codec: codec_id, num_threads: (num_cpus::get() / 2) as _, }) .unwrap(); let start = Instant::now(); - for vp9 in vp9s { - let _ = decoder.decode(&vp9); + for vpx in vpxs { + let _ = decoder.decode(&vpx); let _ = decoder.flush(); } - println!("vp9 decode: {:?}", start.elapsed() / yuv_count as _); + println!( + "{:?} decode: {:?}", + codec_id, + start.elapsed() / yuv_count as _ + ); } #[cfg(feature = "hwcodec")] @@ -267,7 +283,7 @@ mod hw { Some(info.clone()) == best.h264 || Some(info.clone()) == best.h265 } - pub fn vp9_yuv_to_hw_yuv( + pub fn vpx_yuv_to_hw_yuv( yuvs: Vec>, width: usize, height: usize, diff --git a/libs/scrap/src/common/android.rs b/libs/scrap/src/common/android.rs index 8daf8e4bb..36d6a8a9b 100644 --- a/libs/scrap/src/common/android.rs +++ b/libs/scrap/src/common/android.rs @@ -50,6 +50,7 @@ impl crate::TraitCapturer for Capturer { pub enum Frame<'a> { RAW(&'a [u8]), + VP8(&'a [u8]), VP9(&'a [u8]), Empty, } diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 9e4b6fce4..3209933b4 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -1,7 +1,6 @@ -use std::ops::{Deref, DerefMut}; -#[cfg(feature = "hwcodec")] use std::{ collections::HashMap, + ops::{Deref, DerefMut}, sync::{Arc, Mutex}, }; @@ -11,30 +10,31 @@ use crate::hwcodec::*; use crate::mediacodec::{ MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT, }; -use crate::{vpxcodec::*, ImageFormat}; +use crate::{vpxcodec::*, CodecName, ImageFormat}; +#[cfg(not(any(target_os = "android", target_os = "ios")))] +use hbb_common::sysinfo::{System, SystemExt}; use hbb_common::{ anyhow::anyhow, + config::PeerConfig, log, - message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState}, + message_proto::{ + supported_decoding::PreferCodec, video_frame, EncodedVideoFrames, Message, + SupportedDecoding, SupportedEncoding, + }, ResultType, }; #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] -use hbb_common::{ - config::{Config2, PeerConfig}, - lazy_static, - message_proto::video_codec_state::PreferCodec, -}; +use hbb_common::{config::Config2, lazy_static}; -#[cfg(feature = "hwcodec")] lazy_static::lazy_static! { - static ref PEER_DECODER_STATES: Arc>> = Default::default(); + static ref PEER_DECODINGS: Arc>> = Default::default(); + static ref CODEC_NAME: Arc> = Arc::new(Mutex::new(CodecName::VP9)); } -const SCORE_VPX: i32 = 90; #[derive(Debug, Clone)] pub struct HwEncoderConfig { - pub codec_name: String, + pub name: String, pub width: usize, pub height: usize, pub bitrate: i32, @@ -58,10 +58,6 @@ pub trait EncoderApi { fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>; } -pub struct DecoderCfg { - pub vpx: VpxDecoderConfig, -} - pub struct Encoder { pub codec: Box, } @@ -81,7 +77,8 @@ impl DerefMut for Encoder { } pub struct Decoder { - vpx: VpxDecoder, + vp8: VpxDecoder, + vp9: VpxDecoder, #[cfg(feature = "hwcodec")] hw: HwDecoders, #[cfg(feature = "hwcodec")] @@ -91,10 +88,10 @@ pub struct Decoder { } #[derive(Debug, Clone)] -pub enum EncoderUpdate { - State(VideoCodecState), +pub enum EncodingUpdate { + New(SupportedDecoding), Remove, - DisableHwIfNotExist, + NewOnlyVP9, } impl Encoder { @@ -120,172 +117,156 @@ impl Encoder { } } - // TODO - pub fn update_video_encoder(id: i32, update: EncoderUpdate) { + pub fn update(id: i32, update: EncodingUpdate) { + let mut decodings = PEER_DECODINGS.lock().unwrap(); + match update { + EncodingUpdate::New(decoding) => { + decodings.insert(id, decoding); + } + EncodingUpdate::Remove => { + decodings.remove(&id); + } + EncodingUpdate::NewOnlyVP9 => { + decodings.insert( + id, + SupportedDecoding { + ability_vp9: 1, + ..Default::default() + }, + ); + } + } + + let vp8_useable = decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_vp8 > 0); + #[allow(unused_mut)] + let mut h264_name = None; + #[allow(unused_mut)] + let mut h265_name = None; #[cfg(feature = "hwcodec")] { - let mut states = PEER_DECODER_STATES.lock().unwrap(); - match update { - EncoderUpdate::State(state) => { - states.insert(id, state); - } - EncoderUpdate::Remove => { - states.remove(&id); - } - EncoderUpdate::DisableHwIfNotExist => { - if !states.contains_key(&id) { - states.insert(id, VideoCodecState::default()); - } - } + let best = HwEncoder::best(); + let h264_useable = + decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_h264 > 0); + let h265_useable = + decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_h265 > 0); + if h264_useable { + h264_name = best.h264.map_or(None, |c| Some(c.name)); } - let name = HwEncoder::current_name(); - if states.len() > 0 { - let best = HwEncoder::best(); - let enabled_h264 = best.h264.is_some() - && states.len() > 0 - && states.iter().all(|(_, s)| s.score_h264 > 0); - let enabled_h265 = best.h265.is_some() - && states.len() > 0 - && states.iter().all(|(_, s)| s.score_h265 > 0); - - // Preference first - let mut preference = PreferCodec::Auto; - let preferences: Vec<_> = states - .iter() - .filter(|(_, s)| { - s.prefer == PreferCodec::VPX.into() - || s.prefer == PreferCodec::H264.into() && enabled_h264 - || s.prefer == PreferCodec::H265.into() && enabled_h265 - }) - .map(|(_, s)| s.prefer) - .collect(); - if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) { - preference = preferences[0].enum_value_or(PreferCodec::Auto); - } - - match preference { - PreferCodec::VPX => *name.lock().unwrap() = None, - PreferCodec::H264 => { - *name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name)) - } - PreferCodec::H265 => { - *name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name)) - } - PreferCodec::Auto => { - // score encoder - let mut score_vpx = SCORE_VPX; - let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score); - let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score); - - // score decoder - score_vpx += states.iter().map(|s| s.1.score_vpx).sum::(); - if enabled_h264 { - score_h264 += states.iter().map(|s| s.1.score_h264).sum::(); - } - if enabled_h265 { - score_h265 += states.iter().map(|s| s.1.score_h265).sum::(); - } - - if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 { - *name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name)); - } else if enabled_h264 - && score_h264 >= score_vpx - && score_h264 >= score_h265 - { - *name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name)); - } else { - *name.lock().unwrap() = None; - } - } - } - - log::info!( - "connection count:{}, used preference:{:?}, encoder:{:?}", - states.len(), - preference, - name.lock().unwrap() - ) - } else { - *name.lock().unwrap() = None; + if h265_useable { + h265_name = best.h265.map_or(None, |c| Some(c.name)); } } - #[cfg(not(feature = "hwcodec"))] - { - let _ = id; - let _ = update; + + let mut name = CODEC_NAME.lock().unwrap(); + let mut preference = PreferCodec::Auto; + let preferences: Vec<_> = decodings + .iter() + .filter(|(_, s)| { + s.prefer == PreferCodec::VP9.into() + || s.prefer == PreferCodec::VP8.into() && vp8_useable + || s.prefer == PreferCodec::H264.into() && h264_name.is_some() + || s.prefer == PreferCodec::H265.into() && h265_name.is_some() + }) + .map(|(_, s)| s.prefer) + .collect(); + if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) { + preference = preferences[0].enum_value_or(PreferCodec::Auto); } + + #[allow(unused_mut)] + let mut auto_codec = CodecName::VP9; + #[cfg(not(any(target_os = "android", target_os = "ios")))] + if vp8_useable && System::new_all().total_memory() <= 4 * 1024 * 1024 * 1024 { + // 4 Gb + auto_codec = CodecName::VP8 + } + + match preference { + PreferCodec::VP8 => *name = CodecName::VP8, + PreferCodec::VP9 => *name = CodecName::VP9, + PreferCodec::H264 => *name = h264_name.map_or(auto_codec, |c| CodecName::H264(c)), + PreferCodec::H265 => *name = h265_name.map_or(auto_codec, |c| CodecName::H265(c)), + PreferCodec::Auto => *name = auto_codec, + } + + log::info!( + "connection count:{}, used preference:{:?}, encoder:{:?}", + decodings.len(), + preference, + *name + ) } + #[inline] - pub fn current_hw_encoder_name() -> Option { - #[cfg(feature = "hwcodec")] - if enable_hwcodec_option() { - return HwEncoder::current_name().lock().unwrap().clone(); - } else { - return None; - } - #[cfg(not(feature = "hwcodec"))] - return None; + pub fn negotiated_codec() -> CodecName { + CODEC_NAME.lock().unwrap().clone() } - pub fn supported_encoding() -> (bool, bool) { + pub fn supported_encoding() -> SupportedEncoding { + #[allow(unused_mut)] + let mut encoding = SupportedEncoding { + vp8: true, + ..Default::default() + }; #[cfg(feature = "hwcodec")] if enable_hwcodec_option() { let best = HwEncoder::best(); - ( - best.h264.as_ref().map_or(false, |c| c.score > 0), - best.h265.as_ref().map_or(false, |c| c.score > 0), - ) - } else { - (false, false) + encoding.h264 = best.h264.is_some(); + encoding.h265 = best.h265.is_some(); } - #[cfg(not(feature = "hwcodec"))] - (false, false) + encoding } } impl Decoder { - pub fn video_codec_state(_id: &str) -> VideoCodecState { + pub fn supported_decodings(id_for_perfer: Option<&str>) -> SupportedDecoding { + #[allow(unused_mut)] + let mut decoding = SupportedDecoding { + ability_vp8: 1, + ability_vp9: 1, + prefer: id_for_perfer + .map_or(PreferCodec::Auto, |id| Self::codec_preference(id)) + .into(), + ..Default::default() + }; #[cfg(feature = "hwcodec")] if enable_hwcodec_option() { let best = HwDecoder::best(); - return VideoCodecState { - score_vpx: SCORE_VPX, - score_h264: best.h264.map_or(0, |c| c.score), - score_h265: best.h265.map_or(0, |c| c.score), - prefer: Self::codec_preference(_id).into(), - ..Default::default() - }; + decoding.ability_h264 = if best.h264.is_some() { 1 } else { 0 }; + decoding.ability_h265 = if best.h265.is_some() { 1 } else { 0 }; } #[cfg(feature = "mediacodec")] if enable_hwcodec_option() { - let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { - 92 - } else { - 0 - }; - let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { - 94 - } else { - 0 - }; - return VideoCodecState { - score_vpx: SCORE_VPX, - score_h264, - score_h265, - prefer: Self::codec_preference(_id).into(), - ..Default::default() - }; - } - VideoCodecState { - score_vpx: SCORE_VPX, - ..Default::default() + decoding.ability_h264 = + if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 1 + } else { + 0 + }; + decoding.ability_h265 = + if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 1 + } else { + 0 + }; } + decoding } - pub fn new(config: DecoderCfg) -> Decoder { - let vpx = VpxDecoder::new(config.vpx).unwrap(); + pub fn new() -> Decoder { + let vp8 = VpxDecoder::new(VpxDecoderConfig { + codec: VpxVideoCodecId::VP8, + num_threads: (num_cpus::get() / 2) as _, + }) + .unwrap(); + let vp9 = VpxDecoder::new(VpxDecoderConfig { + codec: VpxVideoCodecId::VP9, + num_threads: (num_cpus::get() / 2) as _, + }) + .unwrap(); Decoder { - vpx, + vp8, + vp9, #[cfg(feature = "hwcodec")] hw: if enable_hwcodec_option() { HwDecoder::new_decoders() @@ -310,8 +291,11 @@ impl Decoder { rgb: &mut Vec, ) -> ResultType { match frame { + video_frame::Union::Vp8s(vp8s) => { + Decoder::handle_vpxs_video_frame(&mut self.vp8, vp8s, fmt, rgb) + } video_frame::Union::Vp9s(vp9s) => { - Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, fmt, rgb) + Decoder::handle_vpxs_video_frame(&mut self.vp9, vp9s, fmt, rgb) } #[cfg(feature = "hwcodec")] video_frame::Union::H264s(h264s) => { @@ -349,15 +333,15 @@ impl Decoder { } } - fn handle_vp9s_video_frame( + fn handle_vpxs_video_frame( decoder: &mut VpxDecoder, - vp9s: &EncodedVideoFrames, + vpxs: &EncodedVideoFrames, fmt: (ImageFormat, usize), rgb: &mut Vec, ) -> ResultType { let mut last_frame = Image::new(); - for vp9 in vp9s.frames.iter() { - for frame in decoder.decode(&vp9.data)? { + for vpx in vpxs.frames.iter() { + for frame in decoder.decode(&vpx.data)? { drop(last_frame); last_frame = frame; } @@ -408,14 +392,15 @@ impl Decoder { return Ok(false); } - #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] fn codec_preference(id: &str) -> PreferCodec { let codec = PeerConfig::load(id) .options .get("codec-preference") .map_or("".to_owned(), |c| c.to_owned()); - if codec == "vp9" { - PreferCodec::VPX + if codec == "vp8" { + PreferCodec::VP8 + } else if codec == "vp9" { + PreferCodec::VP9 } else if codec == "h264" { PreferCodec::H264 } else if codec == "h265" { diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 2c69774fb..f4de4bf84 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -7,7 +7,7 @@ use hbb_common::{ anyhow::{anyhow, Context}, bytes::Bytes, config::HwCodecConfig, - lazy_static, log, + log, message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}, ResultType, }; @@ -19,11 +19,6 @@ use hwcodec::{ Quality::{self, *}, RateControl::{self, *}, }; -use std::sync::{Arc, Mutex}; - -lazy_static::lazy_static! { - static ref HW_ENCODER_NAME: Arc>> = Default::default(); -} const CFG_KEY_ENCODER: &str = "bestHwEncoders"; const CFG_KEY_DECODER: &str = "bestHwDecoders"; @@ -49,7 +44,7 @@ impl EncoderApi for HwEncoder { match cfg { EncoderCfg::HW(config) => { let ctx = EncodeContext { - name: config.codec_name.clone(), + name: config.name.clone(), width: config.width as _, height: config.height as _, pixfmt: DEFAULT_PIXFMT, @@ -60,12 +55,12 @@ impl EncoderApi for HwEncoder { quality: DEFAULT_HW_QUALITY, rc: DEFAULT_RC, }; - let format = match Encoder::format_from_name(config.codec_name.clone()) { + let format = match Encoder::format_from_name(config.name.clone()) { Ok(format) => format, Err(_) => { return Err(anyhow!(format!( "failed to get format from name:{}", - config.codec_name + config.name ))) } }; @@ -133,10 +128,6 @@ impl HwEncoder { }) } - pub fn current_name() -> Arc>> { - HW_ENCODER_NAME.clone() - } - pub fn encode(&mut self, bgra: &[u8]) -> ResultType> { match self.pixfmt { AVPixelFormat::AV_PIX_FMT_YUV420P => hw::hw_bgra_to_i420( diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 0ad158cca..26b946401 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -1,4 +1,5 @@ pub use self::vpxcodec::*; +use hbb_common::message_proto::{video_frame, VideoFrame}; cfg_if! { if #[cfg(quartz)] { @@ -92,3 +93,55 @@ pub fn is_cursor_embedded() -> bool { pub fn is_cursor_embedded() -> bool { false } + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum CodecName { + VP8, + VP9, + H264(String), + H265(String), +} + +#[derive(PartialEq, Debug, Clone)] +pub enum CodecFormat { + VP8, + VP9, + H264, + H265, + Unknown, +} + +impl From<&VideoFrame> for CodecFormat { + fn from(it: &VideoFrame) -> Self { + match it.union { + Some(video_frame::Union::Vp8s(_)) => CodecFormat::VP8, + Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9, + Some(video_frame::Union::H264s(_)) => CodecFormat::H264, + Some(video_frame::Union::H265s(_)) => CodecFormat::H265, + _ => CodecFormat::Unknown, + } + } +} + +impl From<&CodecName> for CodecFormat { + fn from(value: &CodecName) -> Self { + match value { + CodecName::VP8 => Self::VP8, + CodecName::VP9 => Self::VP9, + CodecName::H264(_) => Self::H264, + CodecName::H265(_) => Self::H265, + } + } +} + +impl ToString for CodecFormat { + fn to_string(&self) -> String { + match self { + CodecFormat::VP8 => "VP8".into(), + CodecFormat::VP9 => "VP9".into(), + CodecFormat::H264 => "H264".into(), + CodecFormat::H265 => "H265".into(), + CodecFormat::Unknown => "Unknow".into(), + } + } +} diff --git a/libs/scrap/src/common/record.rs b/libs/scrap/src/common/record.rs index 9f38f2d6a..9de70ae14 100644 --- a/libs/scrap/src/common/record.rs +++ b/libs/scrap/src/common/record.rs @@ -1,3 +1,4 @@ +use crate::CodecFormat; #[cfg(feature = "hwcodec")] use hbb_common::anyhow::anyhow; use hbb_common::{ @@ -21,13 +22,6 @@ use webm::mux::{self, Segment, Track, VideoTrack, Writer}; const MIN_SECS: u64 = 1; -#[derive(Debug, Clone, PartialEq)] -pub enum RecordCodecID { - VP9, - H264, - H265, -} - #[derive(Debug, Clone)] pub struct RecorderContext { pub server: bool, @@ -36,7 +30,7 @@ pub struct RecorderContext { pub filename: String, pub width: usize, pub height: usize, - pub codec_id: RecordCodecID, + pub format: CodecFormat, pub tx: Option>, } @@ -55,8 +49,9 @@ impl RecorderContext { } let file = if self.server { "s" } else { "c" }.to_string() + &self.id.clone() - + &chrono::Local::now().format("_%Y%m%d%H%M%S").to_string() - + if self.codec_id == RecordCodecID::VP9 { + + &chrono::Local::now().format("_%Y%m%d%H%M%S_").to_string() + + &self.format.to_string() + + if self.format == CodecFormat::VP9 || self.format == CodecFormat::VP8 { ".webm" } else { ".mp4" @@ -107,8 +102,8 @@ impl DerefMut for Recorder { impl Recorder { pub fn new(mut ctx: RecorderContext) -> ResultType { ctx.set_filename()?; - let recorder = match ctx.codec_id { - RecordCodecID::VP9 => Recorder { + let recorder = match ctx.format { + CodecFormat::VP8 | CodecFormat::VP9 => Recorder { inner: Box::new(WebmRecorder::new(ctx.clone())?), ctx, }, @@ -126,8 +121,8 @@ impl Recorder { fn change(&mut self, mut ctx: RecorderContext) -> ResultType<()> { ctx.set_filename()?; - self.inner = match ctx.codec_id { - RecordCodecID::VP9 => Box::new(WebmRecorder::new(ctx.clone())?), + self.inner = match ctx.format { + CodecFormat::VP8 | CodecFormat::VP9 => Box::new(WebmRecorder::new(ctx.clone())?), #[cfg(feature = "hwcodec")] _ => Box::new(HwRecorder::new(ctx.clone())?), #[cfg(not(feature = "hwcodec"))] @@ -148,10 +143,19 @@ impl Recorder { pub fn write_frame(&mut self, frame: &video_frame::Union) -> ResultType<()> { match frame { - video_frame::Union::Vp9s(vp9s) => { - if self.ctx.codec_id != RecordCodecID::VP9 { + video_frame::Union::Vp8s(vp8s) => { + if self.ctx.format != CodecFormat::VP8 { self.change(RecorderContext { - codec_id: RecordCodecID::VP9, + format: CodecFormat::VP8, + ..self.ctx.clone() + })?; + } + vp8s.frames.iter().map(|f| self.write_video(f)).count(); + } + video_frame::Union::Vp9s(vp9s) => { + if self.ctx.format != CodecFormat::VP9 { + self.change(RecorderContext { + format: CodecFormat::VP9, ..self.ctx.clone() })?; } @@ -159,25 +163,25 @@ impl Recorder { } #[cfg(feature = "hwcodec")] video_frame::Union::H264s(h264s) => { - if self.ctx.codec_id != RecordCodecID::H264 { + if self.ctx.format != CodecFormat::H264 { self.change(RecorderContext { - codec_id: RecordCodecID::H264, + format: CodecFormat::H264, ..self.ctx.clone() })?; } - if self.ctx.codec_id == RecordCodecID::H264 { + if self.ctx.format == CodecFormat::H264 { h264s.frames.iter().map(|f| self.write_video(f)).count(); } } #[cfg(feature = "hwcodec")] video_frame::Union::H265s(h265s) => { - if self.ctx.codec_id != RecordCodecID::H265 { + if self.ctx.format != CodecFormat::H265 { self.change(RecorderContext { - codec_id: RecordCodecID::H265, + format: CodecFormat::H265, ..self.ctx.clone() })?; } - if self.ctx.codec_id == RecordCodecID::H265 { + if self.ctx.format == CodecFormat::H265 { h265s.frames.iter().map(|f| self.write_video(f)).count(); } } @@ -221,7 +225,11 @@ impl RecorderApi for WebmRecorder { ctx.width as _, ctx.height as _, None, - mux::VideoCodecId::VP9, + if ctx.format == CodecFormat::VP9 { + mux::VideoCodecId::VP9 + } else { + mux::VideoCodecId::VP8 + }, ); Ok(WebmRecorder { vt, @@ -279,7 +287,7 @@ impl RecorderApi for HwRecorder { filename: ctx.filename.clone(), width: ctx.width, height: ctx.height, - is265: ctx.codec_id == RecordCodecID::H265, + is265: ctx.format == CodecFormat::H265, framerate: crate::hwcodec::DEFAULT_TIME_BASE[1] as _, }) .map_err(|_| anyhow!("Failed to create hardware muxer"))?; diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index 3df9c0461..4820ea171 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -30,6 +30,7 @@ pub struct VpxEncoder { ctx: vpx_codec_ctx_t, width: usize, height: usize, + id: VpxVideoCodecId, } pub struct VpxDecoder { @@ -97,15 +98,10 @@ impl EncoderApi for VpxEncoder { { match cfg { crate::codec::EncoderCfg::VPX(config) => { - let i; - if cfg!(feature = "VP8") { - i = match config.codec { - VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), - VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_cx()); - } + let i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), + }; let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); @@ -187,12 +183,17 @@ impl EncoderApi for VpxEncoder { VP9E_SET_TILE_COLUMNS as _, 4 as c_int )); + } else if config.codec == VpxVideoCodecId::VP8 { + // https://github.com/webmproject/libvpx/blob/972149cafeb71d6f08df89e91a0130d6a38c4b15/vpx/vp8cx.h#L172 + // https://groups.google.com/a/webmproject.org/g/webm-discuss/c/DJhSrmfQ61M + call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 12,)); } Ok(Self { ctx, width: config.width as _, height: config.height as _, + id: config.codec, }) } _ => Err(anyhow!("encoder type mismatch")), @@ -213,7 +214,7 @@ impl EncoderApi for VpxEncoder { // to-do: flush periodically, e.g. 1 second if frames.len() > 0 { - Ok(VpxEncoder::create_msg(frames)) + Ok(VpxEncoder::create_msg(self.id, frames)) } else { Err(anyhow!("no valid frame")) } @@ -280,13 +281,17 @@ impl VpxEncoder { } #[inline] - fn create_msg(vp9s: Vec) -> Message { + pub fn create_msg(codec_id: VpxVideoCodecId, frames: Vec) -> Message { let mut msg_out = Message::new(); let mut vf = VideoFrame::new(); - vf.set_vp9s(EncodedVideoFrames { - frames: vp9s.into(), + let vpxs = EncodedVideoFrames { + frames: frames.into(), ..Default::default() - }); + }; + match codec_id { + VpxVideoCodecId::VP8 => vf.set_vp8s(vpxs), + VpxVideoCodecId::VP9 => vf.set_vp9s(vpxs), + } msg_out.set_video_frame(vf); msg_out } @@ -382,15 +387,10 @@ impl VpxDecoder { pub fn new(config: VpxDecoderConfig) -> Result { // This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can // cause UB if uninitialized. - let i; - if cfg!(feature = "VP8") { - i = match config.codec { - VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), - VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), - }; - } else { - i = call_vpx_ptr!(vpx_codec_vp9_dx()); - } + let i = match config.codec { + VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()), + VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()), + }; let mut ctx = Default::default(); let cfg = vpx_codec_dec_cfg_t { threads: if config.num_threads == 0 { diff --git a/src/client.rs b/src/client.rs index 2f745b70c..ae93ccfcf 100644 --- a/src/client.rs +++ b/src/client.rs @@ -44,9 +44,9 @@ use hbb_common::{ }; pub use helper::*; use scrap::{ - codec::{Decoder, DecoderCfg}, + codec::Decoder, record::{Recorder, RecorderContext}, - ImageFormat, VpxDecoderConfig, VpxVideoCodecId, + ImageFormat, }; use crate::{ @@ -917,12 +917,7 @@ impl VideoHandler { /// Create a new video handler. pub fn new() -> Self { VideoHandler { - decoder: Decoder::new(DecoderCfg { - vpx: VpxDecoderConfig { - codec: VpxVideoCodecId::VP9, - num_threads: (num_cpus::get() / 2) as _, - }, - }), + decoder: Decoder::new(), rgb: Default::default(), recorder: Default::default(), record: false, @@ -954,12 +949,7 @@ impl VideoHandler { /// Reset the decoder. pub fn reset(&mut self) { - self.decoder = Decoder::new(DecoderCfg { - vpx: VpxDecoderConfig { - codec: VpxVideoCodecId::VP9, - num_threads: 1, - }, - }); + self.decoder = Decoder::new(); } /// Start or stop screen record. @@ -973,7 +963,7 @@ impl VideoHandler { filename: "".to_owned(), width: w as _, height: h as _, - codec_id: scrap::record::RecordCodecID::VP9, + format: scrap::CodecFormat::VP9, tx: None, }) .map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r)))); @@ -999,7 +989,7 @@ pub struct LoginConfigHandler { pub conn_id: i32, features: Option, session_id: u64, - pub supported_encoding: Option<(bool, bool)>, + pub supported_encoding: SupportedEncoding, pub restarting_remote_device: bool, pub force_relay: bool, pub direct: Option, @@ -1047,7 +1037,7 @@ impl LoginConfigHandler { self.remember = !config.password.is_empty(); self.config = config; self.session_id = rand::random(); - self.supported_encoding = None; + self.supported_encoding = Default::default(); self.restarting_remote_device = false; self.force_relay = !self.get_option("force-always-relay").is_empty() || force_relay; self.direct = None; @@ -1331,8 +1321,8 @@ impl LoginConfigHandler { msg.disable_clipboard = BoolOption::Yes.into(); n += 1; } - let state = Decoder::video_codec_state(&self.id); - msg.video_codec_state = hbb_common::protobuf::MessageField::some(state); + msg.supported_decoding = + hbb_common::protobuf::MessageField::some(Decoder::supported_decodings(Some(&self.id))); n += 1; if n > 0 { @@ -1565,10 +1555,7 @@ impl LoginConfigHandler { self.conn_id = pi.conn_id; // no matter if change, for update file time self.save_config(config); - #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] - { - self.supported_encoding = Some((pi.encoding.h264, pi.encoding.h265)); - } + self.supported_encoding = pi.encoding.clone().unwrap_or_default(); } pub fn get_remote_dir(&self) -> String { @@ -1626,10 +1613,10 @@ impl LoginConfigHandler { } pub fn change_prefer_codec(&self) -> Message { - let state = scrap::codec::Decoder::video_codec_state(&self.id); + let decoding = scrap::codec::Decoder::supported_decodings(Some(&self.id)); let mut misc = Misc::new(); misc.set_option(OptionMessage { - video_codec_state: hbb_common::protobuf::MessageField::some(state), + supported_decoding: hbb_common::protobuf::MessageField::some(decoding), ..Default::default() }); let mut msg_out = Message::new(); diff --git a/src/client/helper.rs b/src/client/helper.rs index a9696a8e8..61844d908 100644 --- a/src/client/helper.rs +++ b/src/client/helper.rs @@ -1,37 +1,8 @@ use hbb_common::{ get_time, - message_proto::{video_frame, Message, VideoFrame, VoiceCallRequest, VoiceCallResponse}, + message_proto::{Message, VoiceCallRequest, VoiceCallResponse}, }; - -#[derive(PartialEq, Debug, Clone)] -pub enum CodecFormat { - VP9, - H264, - H265, - Unknown, -} - -impl From<&VideoFrame> for CodecFormat { - fn from(it: &VideoFrame) -> Self { - match it.union { - Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9, - Some(video_frame::Union::H264s(_)) => CodecFormat::H264, - Some(video_frame::Union::H265s(_)) => CodecFormat::H265, - _ => CodecFormat::Unknown, - } - } -} - -impl ToString for CodecFormat { - fn to_string(&self) -> String { - match self { - CodecFormat::VP9 => "VP9".into(), - CodecFormat::H264 => "H264".into(), - CodecFormat::H265 => "H265".into(), - CodecFormat::Unknown => "Unknow".into(), - } - } -} +use scrap::CodecFormat; #[derive(Debug, Default)] pub struct QualityStatus { diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index b0bddc82e..0f662e015 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -29,10 +29,10 @@ use hbb_common::tokio::{ time::{self, Duration, Instant, Interval}, }; use hbb_common::{allow_err, fs, get_time, log, message_proto::*, Stream}; +use scrap::CodecFormat; use crate::client::{ - new_voice_call_request, Client, CodecFormat, MediaData, MediaSender, QualityStatus, MILLI1, - SEC30, + new_voice_call_request, Client, MediaData, MediaSender, QualityStatus, MILLI1, SEC30, }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::{self, update_clipboard}; @@ -817,11 +817,10 @@ impl Remote { } fn contains_key_frame(vf: &VideoFrame) -> bool { + use video_frame::Union::*; match &vf.union { Some(vf) => match vf { - video_frame::Union::Vp9s(f) => f.frames.iter().any(|e| e.key), - video_frame::Union::H264s(f) => f.frames.iter().any(|e| e.key), - video_frame::Union::H265s(f) => f.frames.iter().any(|e| e.key), + Vp8s(f) | Vp9s(f) | H264s(f) | H265s(f) => f.frames.iter().any(|e| e.key), _ => false, }, None => false, diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index aee486b94..1d965ebc5 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -969,6 +969,13 @@ pub fn main_has_hwcodec() -> SyncReturn { SyncReturn(has_hwcodec()) } +pub fn main_supported_hwdecodings() -> SyncReturn { + let decoding = supported_hwdecodings(); + let msg = HashMap::from([("h264", decoding.0), ("h265", decoding.1)]); + + SyncReturn(serde_json::ser::to_string(&msg).unwrap_or("".to_owned())) +} + pub fn main_is_root() -> bool { is_root() } @@ -1054,10 +1061,10 @@ pub fn session_send_note(id: String, note: String) { } } -pub fn session_supported_hwcodec(id: String) -> String { +pub fn session_alternative_codecs(id: String) -> String { if let Some(session) = SESSIONS.read().unwrap().get(&id) { - let (h264, h265) = session.supported_hwcodec(); - let msg = HashMap::from([("h264", h264), ("h265", h265)]); + let (vp8, h264, h265) = session.alternative_codecs(); + let msg = HashMap::from([("vp8", vp8), ("h264", h264), ("h265", h265)]); serde_json::ser::to_string(&msg).unwrap_or("".to_owned()) } else { String::new() diff --git a/src/server/connection.rs b/src/server/connection.rs index 725ff43ee..539d96ddd 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -533,7 +533,7 @@ impl Connection { let _ = privacy_mode::turn_off_privacy(0); } video_service::notify_video_frame_fetched(id, None); - scrap::codec::Encoder::update_video_encoder(id, scrap::codec::EncoderUpdate::Remove); + scrap::codec::Encoder::update(id, scrap::codec::EncodingUpdate::Remove); video_service::VIDEO_QOS.lock().unwrap().reset(); if conn.authorized { password::update_temporary_password(); @@ -860,16 +860,7 @@ impl Connection { } } - #[cfg(feature = "hwcodec")] - { - let (h264, h265) = scrap::codec::Encoder::supported_encoding(); - pi.encoding = Some(SupportedEncoding { - h264, - h265, - ..Default::default() - }) - .into(); - } + pi.encoding = Some(scrap::codec::Encoder::supported_encoding()).into(); if self.port_forward_socket.is_some() { let mut msg_out = Message::new(); @@ -1145,21 +1136,21 @@ impl Connection { self.lr = lr.clone(); if let Some(o) = lr.option.as_ref() { self.options_in_login = Some(o.clone()); - if let Some(q) = o.video_codec_state.clone().take() { - scrap::codec::Encoder::update_video_encoder( + if let Some(q) = o.supported_decoding.clone().take() { + scrap::codec::Encoder::update( self.inner.id(), - scrap::codec::EncoderUpdate::State(q), + scrap::codec::EncodingUpdate::New(q), ); } else { - scrap::codec::Encoder::update_video_encoder( + scrap::codec::Encoder::update( self.inner.id(), - scrap::codec::EncoderUpdate::DisableHwIfNotExist, + scrap::codec::EncodingUpdate::NewOnlyVP9, ); } } else { - scrap::codec::Encoder::update_video_encoder( + scrap::codec::Encoder::update( self.inner.id(), - scrap::codec::EncoderUpdate::DisableHwIfNotExist, + scrap::codec::EncodingUpdate::NewOnlyVP9, ); } self.video_ack_required = lr.video_ack_required; @@ -1758,11 +1749,8 @@ impl Connection { .unwrap() .update_user_fps(o.custom_fps as _); } - if let Some(q) = o.video_codec_state.clone().take() { - scrap::codec::Encoder::update_video_encoder( - self.inner.id(), - scrap::codec::EncoderUpdate::State(q), - ); + if let Some(q) = o.supported_decoding.clone().take() { + scrap::codec::Encoder::update(self.inner.id(), scrap::codec::EncodingUpdate::New(q)); } if let Ok(q) = o.lock_after_session_end.enum_value() { if q != BoolOption::NotSet { diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 205d0584c..691ca4abe 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -31,7 +31,7 @@ use scrap::{ codec::{Encoder, EncoderCfg, HwEncoderConfig}, record::{Recorder, RecorderContext}, vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, - Display, TraitCapturer, + CodecName, Display, TraitCapturer, }; #[cfg(windows)] use std::sync::Once; @@ -468,21 +468,29 @@ fn run(sp: GenericService) -> ResultType<()> { drop(video_qos); log::info!("init bitrate={}, abr enabled:{}", bitrate, abr); - let encoder_cfg = match Encoder::current_hw_encoder_name() { - Some(codec_name) => EncoderCfg::HW(HwEncoderConfig { - codec_name, - width: c.width, - height: c.height, - bitrate: bitrate as _, - }), - None => EncoderCfg::VPX(VpxEncoderConfig { - width: c.width as _, - height: c.height as _, - timebase: [1, 1000], // Output timestamp precision - bitrate, - codec: VpxVideoCodecId::VP9, - num_threads: (num_cpus::get() / 2) as _, - }), + let encoder_cfg = match Encoder::negotiated_codec() { + scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => { + EncoderCfg::HW(HwEncoderConfig { + name, + width: c.width, + height: c.height, + bitrate: bitrate as _, + }) + } + name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => { + EncoderCfg::VPX(VpxEncoderConfig { + width: c.width as _, + height: c.height as _, + timebase: [1, 1000], // Output timestamp precision + bitrate, + codec: if name == scrap::CodecName::VP8 { + VpxVideoCodecId::VP8 + } else { + VpxVideoCodecId::VP9 + }, + num_threads: (num_cpus::get() / 2) as _, + }) + } }; let mut encoder; @@ -526,7 +534,7 @@ fn run(sp: GenericService) -> ResultType<()> { let mut try_gdi = 1; #[cfg(windows)] log::info!("gdi: {}", c.is_gdi()); - let codec_name = Encoder::current_hw_encoder_name(); + let codec_name = Encoder::negotiated_codec(); let recorder = get_recorder(c.width, c.height, &codec_name); #[cfg(windows)] start_uac_elevation_check(); @@ -557,7 +565,7 @@ fn run(sp: GenericService) -> ResultType<()> { *SWITCH.lock().unwrap() = true; bail!("SWITCH"); } - if codec_name != Encoder::current_hw_encoder_name() { + if codec_name != Encoder::negotiated_codec() { bail!("SWITCH"); } #[cfg(windows)] @@ -603,8 +611,14 @@ fn run(sp: GenericService) -> ResultType<()> { let time = now - start; let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; match frame { + scrap::Frame::VP8(data) => { + let send_conn_ids = + handle_one_frame_encoded(VpxVideoCodecId::VP8, &sp, data, ms)?; + frame_controller.set_send(now, send_conn_ids); + } scrap::Frame::VP9(data) => { - let send_conn_ids = handle_one_frame_encoded(&sp, data, ms)?; + let send_conn_ids = + handle_one_frame_encoded(VpxVideoCodecId::VP9, &sp, data, ms)?; frame_controller.set_send(now, send_conn_ids); } scrap::Frame::RAW(data) => { @@ -717,12 +731,11 @@ fn run(sp: GenericService) -> ResultType<()> { fn get_recorder( width: usize, height: usize, - codec_name: &Option, + codec_name: &CodecName, ) -> Arc>> { #[cfg(not(target_os = "ios"))] let recorder = if !Config::get_option("allow-auto-record-incoming").is_empty() { use crate::hbbs_http::record_upload; - use scrap::record::RecordCodecID::*; let tx = if record_upload::is_enable() { let (tx, rx) = std::sync::mpsc::channel(); @@ -731,16 +744,6 @@ fn get_recorder( } else { None }; - let codec_id = match codec_name { - Some(name) => { - if name.contains("264") { - H264 - } else { - H265 - } - } - None => VP9, - }; Recorder::new(RecorderContext { server: true, id: Config::get_id(), @@ -748,7 +751,7 @@ fn get_recorder( filename: "".to_owned(), width, height, - codec_id, + format: codec_name.into(), tx, }) .map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r)))) @@ -775,19 +778,6 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu Ok(()) } -#[inline] -#[cfg(any(target_os = "android", target_os = "ios"))] -fn create_msg(vp9s: Vec) -> Message { - let mut msg_out = Message::new(); - let mut vf = VideoFrame::new(); - vf.set_vp9s(EncodedVideoFrames { - frames: vp9s.into(), - ..Default::default() - }); - msg_out.set_video_frame(vf); - msg_out -} - #[inline] fn handle_one_frame( sp: &GenericService, @@ -820,6 +810,7 @@ fn handle_one_frame( #[inline] #[cfg(any(target_os = "android", target_os = "ios"))] pub fn handle_one_frame_encoded( + codec: VpxVideoCodecId, sp: &GenericService, frame: &[u8], ms: i64, @@ -831,13 +822,13 @@ pub fn handle_one_frame_encoded( } Ok(()) })?; - let vp9_frame = EncodedVideoFrame { + let vpx_frame = EncodedVideoFrame { data: frame.to_vec().into(), key: true, pts: ms, ..Default::default() }; - let send_conn_ids = sp.send_video_frame(create_msg(vec![vp9_frame])); + let send_conn_ids = sp.send_video_frame(scrap::VpxEncoder::create_msg(codec, vec![vpx_frame])); Ok(send_conn_ids) } diff --git a/src/ui/header.tis b/src/ui/header.tis index 257ba417e..af4f1e349 100644 --- a/src/ui/header.tis +++ b/src/ui/header.tis @@ -161,8 +161,8 @@ class Header: Reactor.Component { } function renderDisplayPop() { - var codecs = handler.supported_hwcodec(); - var show_codec = handler.has_hwcodec() && (codecs[0] || codecs[1]); + var codecs = handler.alternative_codecs(); + var show_codec = codecs[0] || codecs[1] || codecs[2]; var cursor_embedded = false; if ((pi.displays || []).length > 0) { @@ -186,9 +186,10 @@ class Header: Reactor.Component { {show_codec ?
  • {svg_checkmark}Auto
    • + {codecs[0] ?
  • {svg_checkmark}VP8
    • : ""}
  • {svg_checkmark}VP9
    • - {codecs[0] ?
  • {svg_checkmark}H264
    • : ""} - {codecs[1] ?
  • {svg_checkmark}H265
    • : ""} + {codecs[1] ?
  • {svg_checkmark}H264
    • : ""} + {codecs[2] ?
  • {svg_checkmark}H265
    • : ""}
    : ""}
    {!cursor_embedded &&
  • {svg_checkmark}{translate('Show remote cursor')}
    • } diff --git a/src/ui/remote.rs b/src/ui/remote.rs index 68decf955..fc878cf1f 100644 --- a/src/ui/remote.rs +++ b/src/ui/remote.rs @@ -20,7 +20,6 @@ use hbb_common::{ use crate::{ client::*, - ui_interface::has_hwcodec, ui_session_interface::{InvokeUiSession, Session}, }; @@ -197,7 +196,14 @@ impl InvokeUiSession for SciterHandler { self.call("confirmDeleteFiles", &make_args!(id, i, name)); } - fn override_file_confirm(&self, id: i32, file_num: i32, to: String, is_upload: bool, is_identical: bool) { + fn override_file_confirm( + &self, + id: i32, + file_num: i32, + to: String, + is_upload: bool, + is_identical: bool, + ) { self.call( "overrideFileConfirm", &make_args!(id, file_num, to, is_upload, is_identical), @@ -451,8 +457,7 @@ impl sciter::EventHandler for SciterSession { fn set_write_override(i32, i32, bool, bool, bool); fn get_keyboard_mode(); fn save_keyboard_mode(String); - fn has_hwcodec(); - fn supported_hwcodec(); + fn alternative_codecs(); fn change_prefer_codec(); fn restart_remote_device(); fn request_voice_call(); @@ -504,10 +509,6 @@ impl SciterSession { v } - fn has_hwcodec(&self) -> bool { - has_hwcodec() - } - pub fn t(&self, name: String) -> String { crate::client::translate(name) } @@ -516,9 +517,10 @@ impl SciterSession { super::get_icon() } - fn supported_hwcodec(&self) -> Value { - let (h264, h265) = self.0.supported_hwcodec(); + fn alternative_codecs(&self) -> Value { + let (vp8, h264, h265) = self.0.alternative_codecs(); let mut v = Value::array(0); + v.push(vp8); v.push(h264); v.push(h265); v diff --git a/src/ui_interface.rs b/src/ui_interface.rs index 11357be4a..26d470fe0 100644 --- a/src/ui_interface.rs +++ b/src/ui_interface.rs @@ -752,6 +752,13 @@ pub fn has_hwcodec() -> bool { return true; } +#[cfg(feature = "flutter")] +#[inline] +pub fn supported_hwdecodings() -> (bool, bool) { + let decoding = scrap::codec::Decoder::supported_decodings(None); + (decoding.ability_h264 > 0, decoding.ability_h265 > 0) +} + #[cfg(not(any(target_os = "android", target_os = "ios")))] #[inline] pub fn is_root() -> bool { diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index f89be4879..3dee89a6e 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -216,24 +216,16 @@ impl Session { true } - pub fn supported_hwcodec(&self) -> (bool, bool) { - #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] - { - let decoder = scrap::codec::Decoder::video_codec_state(&self.id); - let mut h264 = decoder.score_h264 > 0; - let mut h265 = decoder.score_h265 > 0; - let (encoding_264, encoding_265) = self - .lc - .read() - .unwrap() - .supported_encoding - .unwrap_or_default(); - h264 = h264 && encoding_264; - h265 = h265 && encoding_265; - return (h264, h265); - } - #[allow(unreachable_code)] - (false, false) + pub fn alternative_codecs(&self) -> (bool, bool, bool) { + let decoder = scrap::codec::Decoder::supported_decodings(None); + let mut vp8 = decoder.ability_vp8 > 0; + let mut h264 = decoder.ability_h264 > 0; + let mut h265 = decoder.ability_h265 > 0; + let enc = &self.lc.read().unwrap().supported_encoding; + vp8 = vp8 && enc.vp8; + h264 = h264 && enc.h264; + h265 = h265 && enc.h265; + (vp8, h264, h265) } pub fn change_prefer_codec(&self) {