From 78748271ac329a63fea837d772a7f7da4f462f89 Mon Sep 17 00:00:00 2001
From: 21pages <pages21@163.com>
Date: Fri, 31 Mar 2023 16:10:52 +0800
Subject: [PATCH] vp8
Signed-off-by: 21pages <pages21@163.com>
---
.../desktop/pages/desktop_setting_page.dart | 42 ++-
.../lib/desktop/widgets/remote_toolbar.dart | 26 +-
flutter/lib/mobile/pages/remote_page.dart | 5 +-
libs/hbb_common/protos/message.proto | 16 +-
libs/hbb_common/src/config.rs | 5 +-
libs/scrap/examples/benchmark.rs | 46 ++-
libs/scrap/src/common/android.rs | 1 +
libs/scrap/src/common/codec.rs | 319 +++++++++---------
libs/scrap/src/common/hwcodec.rs | 17 +-
libs/scrap/src/common/mod.rs | 53 +++
libs/scrap/src/common/record.rs | 58 ++--
libs/scrap/src/common/vpxcodec.rs | 46 +--
src/client.rs | 37 +-
src/client/helper.rs | 33 +-
src/client/io_loop.rs | 9 +-
src/flutter_ffi.rs | 13 +-
src/server/connection.rs | 34 +-
src/server/video_service.rs | 85 +++--
src/ui/header.tis | 9 +-
src/ui/remote.rs | 22 +-
src/ui_interface.rs | 7 +
src/ui_session_interface.rs | 28 +-
22 files changed, 470 insertions(+), 441 deletions(-)
diff --git a/flutter/lib/desktop/pages/desktop_setting_page.dart b/flutter/lib/desktop/pages/desktop_setting_page.dart
index 66ef83d31..74d51407c 100644
--- a/flutter/lib/desktop/pages/desktop_setting_page.dart
+++ b/flutter/lib/desktop/pages/desktop_setting_page.dart
@@ -1258,9 +1258,6 @@ class _DisplayState extends State<_Display> {
}
Widget codec(BuildContext context) {
- if (!bind.mainHasHwcodec()) {
- return Offstage();
- }
final key = 'codec-preference';
onChanged(String value) async {
await bind.mainSetUserDefaultOption(key: key, value: value);
@@ -1268,28 +1265,45 @@ class _DisplayState extends State<_Display> {
}
final groupValue = bind.mainGetUserDefaultOption(key: key);
-
+ var hwRadios = [];
+ try {
+ final Map codecsJson = jsonDecode(bind.mainSupportedHwdecodings());
+ final h264 = codecsJson['h264'] ?? false;
+ final h265 = codecsJson['h265'] ?? false;
+ if (h264) {
+ hwRadios.add(_Radio(context,
+ value: 'h264',
+ groupValue: groupValue,
+ label: 'H264',
+ onChanged: onChanged));
+ }
+ if (h265) {
+ hwRadios.add(_Radio(context,
+ value: 'h265',
+ groupValue: groupValue,
+ label: 'H265',
+ onChanged: onChanged));
+ }
+ } catch (e) {
+ debugPrint("failed to parse supported hwdecodings, err=$e");
+ }
return _Card(title: 'Default Codec', children: [
_Radio(context,
value: 'auto',
groupValue: groupValue,
label: 'Auto',
onChanged: onChanged),
+ _Radio(context,
+ value: 'vp8',
+ groupValue: groupValue,
+ label: 'VP8',
+ onChanged: onChanged),
_Radio(context,
value: 'vp9',
groupValue: groupValue,
label: 'VP9',
onChanged: onChanged),
- _Radio(context,
- value: 'h264',
- groupValue: groupValue,
- label: 'H264',
- onChanged: onChanged),
- _Radio(context,
- value: 'h265',
- groupValue: groupValue,
- label: 'H265',
- onChanged: onChanged),
+ ...hwRadios,
]);
}
diff --git a/flutter/lib/desktop/widgets/remote_toolbar.dart b/flutter/lib/desktop/widgets/remote_toolbar.dart
index 0ef8674ef..cb4b6d644 100644
--- a/flutter/lib/desktop/widgets/remote_toolbar.dart
+++ b/flutter/lib/desktop/widgets/remote_toolbar.dart
@@ -1349,29 +1349,30 @@ class _DisplayMenuState extends State<_DisplayMenu> {
codec() {
return futureBuilder(future: () async {
- final supportedHwcodec =
- await bind.sessionSupportedHwcodec(id: widget.id);
+ final alternativeCodecs =
+ await bind.sessionAlternativeCodecs(id: widget.id);
final codecPreference =
await bind.sessionGetOption(id: widget.id, arg: 'codec-preference') ??
'';
return {
- 'supportedHwcodec': supportedHwcodec,
+ 'alternativeCodecs': alternativeCodecs,
'codecPreference': codecPreference
};
}(), hasData: (data) {
final List<bool> codecs = [];
try {
- final Map codecsJson = jsonDecode(data['supportedHwcodec']);
+ final Map codecsJson = jsonDecode(data['alternativeCodecs']);
+ final vp8 = codecsJson['vp8'] ?? false;
final h264 = codecsJson['h264'] ?? false;
final h265 = codecsJson['h265'] ?? false;
+ codecs.add(vp8);
codecs.add(h264);
codecs.add(h265);
} catch (e) {
debugPrint("Show Codec Preference err=$e");
}
- final visible = bind.mainHasHwcodec() &&
- codecs.length == 2 &&
- (codecs[0] || codecs[1]);
+ final visible =
+ codecs.length == 3 && (codecs[0] || codecs[1] || codecs[2]);
if (!visible) return Offstage();
final groupValue = data['codecPreference'] as String;
onChanged(String? value) async {
@@ -1392,6 +1393,13 @@ class _DisplayMenuState extends State<_DisplayMenu> {
onChanged: onChanged,
ffi: widget.ffi,
),
+ _RadioMenuButton<String>(
+ child: Text(translate('VP8')),
+ value: 'vp8',
+ groupValue: groupValue,
+ onChanged: codecs[0] ? onChanged : null,
+ ffi: widget.ffi,
+ ),
_RadioMenuButton<String>(
child: Text(translate('VP9')),
value: 'vp9',
@@ -1403,14 +1411,14 @@ class _DisplayMenuState extends State<_DisplayMenu> {
child: Text(translate('H264')),
value: 'h264',
groupValue: groupValue,
- onChanged: codecs[0] ? onChanged : null,
+ onChanged: codecs[1] ? onChanged : null,
ffi: widget.ffi,
),
_RadioMenuButton<String>(
child: Text(translate('H265')),
value: 'h265',
groupValue: groupValue,
- onChanged: codecs[1] ? onChanged : null,
+ onChanged: codecs[2] ? onChanged : null,
ffi: widget.ffi,
),
]);
diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart
index 083cdcd1c..74a327c27 100644
--- a/flutter/lib/mobile/pages/remote_page.dart
+++ b/flutter/lib/mobile/pages/remote_page.dart
@@ -973,9 +973,11 @@ void showOptions(
if (hasHwcodec) {
try {
final Map codecsJson =
- jsonDecode(await bind.sessionSupportedHwcodec(id: id));
+ jsonDecode(await bind.sessionAlternativeCodecs(id: id));
+ final vp8 = codecsJson['vp8'] ?? false;
final h264 = codecsJson['h264'] ?? false;
final h265 = codecsJson['h265'] ?? false;
+ codecs.add(vp8);
codecs.add(h264);
codecs.add(h265);
} catch (e) {
@@ -1044,6 +1046,7 @@ void showOptions(
if (hasHwcodec && codecs.length == 2 && (codecs[0] || codecs[1])) {
radios.addAll([
getRadio(translate('Auto'), 'auto', codec, setCodec),
+ getRadio('VP8', 'vp8', codec, setCodec),
getRadio('VP9', 'vp9', codec, setCodec),
]);
if (codecs[0]) {
diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto
index 6295c160b..321747e3a 100644
--- a/libs/hbb_common/protos/message.proto
+++ b/libs/hbb_common/protos/message.proto
@@ -24,6 +24,7 @@ message VideoFrame {
YUV yuv = 8;
EncodedVideoFrames h264s = 10;
EncodedVideoFrames h265s = 11;
+ EncodedVideoFrames vp8s = 12;
}
}
@@ -76,6 +77,7 @@ message Features {
message SupportedEncoding {
bool h264 = 1;
bool h265 = 2;
+ bool vp8 = 3;
}
message PeerInfo {
@@ -457,18 +459,20 @@ enum ImageQuality {
Best = 4;
}
-message VideoCodecState {
+message SupportedDecoding {
enum PreferCodec {
Auto = 0;
- VPX = 1;
+ VP9 = 1;
H264 = 2;
H265 = 3;
+ VP8 = 4;
}
- int32 score_vpx = 1;
- int32 score_h264 = 2;
- int32 score_h265 = 3;
+ int32 ability_vp9 = 1;
+ int32 ability_h264 = 2;
+ int32 ability_h265 = 3;
PreferCodec prefer = 4;
+ int32 ability_vp8 = 5;
}
message OptionMessage {
@@ -486,7 +490,7 @@ message OptionMessage {
BoolOption disable_audio = 7;
BoolOption disable_clipboard = 8;
BoolOption enable_file_transfer = 9;
- VideoCodecState video_codec_state = 10;
+ SupportedDecoding supported_decoding = 10;
int32 custom_fps = 11;
BoolOption disable_keyboard = 12;
}
diff --git a/libs/hbb_common/src/config.rs b/libs/hbb_common/src/config.rs
index 6a823c7b7..960074a8f 100644
--- a/libs/hbb_common/src/config.rs
+++ b/libs/hbb_common/src/config.rs
@@ -917,7 +917,8 @@ impl PeerConfig {
store = store || store2;
for opt in ["rdp_password", "os-password"] {
if let Some(v) = config.options.get_mut(opt) {
- let (encrypted, _, store2) = decrypt_str_or_original(v, PASSWORD_ENC_VERSION);
+ let (encrypted, _, store2) =
+ decrypt_str_or_original(v, PASSWORD_ENC_VERSION);
*v = encrypted;
store = store || store2;
}
@@ -1356,7 +1357,7 @@ impl UserDefaultConfig {
"view_style" => self.get_string(key, "original", vec!["adaptive"]),
"scroll_style" => self.get_string(key, "scrollauto", vec!["scrollbar"]),
"image_quality" => self.get_string(key, "balanced", vec!["best", "low", "custom"]),
- "codec-preference" => self.get_string(key, "auto", vec!["vp9", "h264", "h265"]),
+ "codec-preference" => self.get_string(key, "auto", vec!["vp8", "vp9", "h264", "h265"]),
"custom_image_quality" => self.get_double_string(key, 50.0, 10.0, 100.0),
"custom-fps" => self.get_double_string(key, 30.0, 10.0, 120.0),
_ => self
diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs
index 003830f95..ba8dec9f2 100644
--- a/libs/scrap/examples/benchmark.rs
+++ b/libs/scrap/examples/benchmark.rs
@@ -3,7 +3,8 @@ use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV};
use scrap::{
codec::{EncoderApi, EncoderCfg},
Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig,
- VpxVideoCodecId, STRIDE_ALIGN,
+ VpxVideoCodecId::{self, *},
+ STRIDE_ALIGN,
};
use std::{io::Write, time::Instant};
@@ -49,7 +50,7 @@ fn main() {
"benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}",
width, height, bitrate_k, args.flag_hw_pixfmt
);
- test_vp9(&yuvs, width, height, bitrate_k, yuv_count);
+ [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count));
#[cfg(feature = "hwcodec")]
{
use hwcodec::AVPixelFormat;
@@ -57,7 +58,7 @@ fn main() {
Pixfmt::I420 => AVPixelFormat::AV_PIX_FMT_YUV420P,
Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12,
};
- let yuvs = hw::vp9_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt);
+ let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt);
hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt);
}
}
@@ -87,13 +88,20 @@ fn capture_yuv(yuv_count: usize) -> (Vec<Vec<u8>>, usize, usize) {
}
}
-fn test_vp9(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) {
+fn test_vpx(
+ codec_id: VpxVideoCodecId,
+ yuvs: &Vec<Vec<u8>>,
+ width: usize,
+ height: usize,
+ bitrate_k: usize,
+ yuv_count: usize,
+) {
let config = EncoderCfg::VPX(VpxEncoderConfig {
width: width as _,
height: height as _,
timebase: [1, 1000],
bitrate: bitrate_k as _,
- codec: VpxVideoCodecId::VP9,
+ codec: codec_id,
num_threads: (num_cpus::get() / 2) as _,
});
let mut encoder = VpxEncoder::new(config).unwrap();
@@ -104,35 +112,43 @@ fn test_vp9(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, bitrate_k: usize,
.unwrap();
let _ = encoder.flush().unwrap();
}
- println!("vp9 encode: {:?}", start.elapsed() / yuv_count as _);
+ println!(
+ "{:?} encode: {:?}",
+ codec_id,
+ start.elapsed() / yuv_count as _
+ );
// prepare data separately
- let mut vp9s = vec![];
+ let mut vpxs = vec![];
let start = Instant::now();
for yuv in yuvs {
for ref frame in encoder
.encode(start.elapsed().as_millis() as _, yuv, STRIDE_ALIGN)
.unwrap()
{
- vp9s.push(frame.data.to_vec());
+ vpxs.push(frame.data.to_vec());
}
for ref frame in encoder.flush().unwrap() {
- vp9s.push(frame.data.to_vec());
+ vpxs.push(frame.data.to_vec());
}
}
- assert_eq!(vp9s.len(), yuv_count);
+ assert_eq!(vpxs.len(), yuv_count);
let mut decoder = VpxDecoder::new(VpxDecoderConfig {
- codec: VpxVideoCodecId::VP9,
+ codec: codec_id,
num_threads: (num_cpus::get() / 2) as _,
})
.unwrap();
let start = Instant::now();
- for vp9 in vp9s {
- let _ = decoder.decode(&vp9);
+ for vpx in vpxs {
+ let _ = decoder.decode(&vpx);
let _ = decoder.flush();
}
- println!("vp9 decode: {:?}", start.elapsed() / yuv_count as _);
+ println!(
+ "{:?} decode: {:?}",
+ codec_id,
+ start.elapsed() / yuv_count as _
+ );
}
#[cfg(feature = "hwcodec")]
@@ -267,7 +283,7 @@ mod hw {
Some(info.clone()) == best.h264 || Some(info.clone()) == best.h265
}
- pub fn vp9_yuv_to_hw_yuv(
+ pub fn vpx_yuv_to_hw_yuv(
yuvs: Vec<Vec<u8>>,
width: usize,
height: usize,
diff --git a/libs/scrap/src/common/android.rs b/libs/scrap/src/common/android.rs
index 8daf8e4bb..36d6a8a9b 100644
--- a/libs/scrap/src/common/android.rs
+++ b/libs/scrap/src/common/android.rs
@@ -50,6 +50,7 @@ impl crate::TraitCapturer for Capturer {
pub enum Frame<'a> {
RAW(&'a [u8]),
+ VP8(&'a [u8]),
VP9(&'a [u8]),
Empty,
}
diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs
index 9e4b6fce4..3209933b4 100644
--- a/libs/scrap/src/common/codec.rs
+++ b/libs/scrap/src/common/codec.rs
@@ -1,7 +1,6 @@
-use std::ops::{Deref, DerefMut};
-#[cfg(feature = "hwcodec")]
use std::{
collections::HashMap,
+ ops::{Deref, DerefMut},
sync::{Arc, Mutex},
};
@@ -11,30 +10,31 @@ use crate::hwcodec::*;
use crate::mediacodec::{
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
};
-use crate::{vpxcodec::*, ImageFormat};
+use crate::{vpxcodec::*, CodecName, ImageFormat};
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+use hbb_common::sysinfo::{System, SystemExt};
use hbb_common::{
anyhow::anyhow,
+ config::PeerConfig,
log,
- message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState},
+ message_proto::{
+ supported_decoding::PreferCodec, video_frame, EncodedVideoFrames, Message,
+ SupportedDecoding, SupportedEncoding,
+ },
ResultType,
};
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
-use hbb_common::{
- config::{Config2, PeerConfig},
- lazy_static,
- message_proto::video_codec_state::PreferCodec,
-};
+use hbb_common::{config::Config2, lazy_static};
-#[cfg(feature = "hwcodec")]
lazy_static::lazy_static! {
- static ref PEER_DECODER_STATES: Arc<Mutex<HashMap<i32, VideoCodecState>>> = Default::default();
+ static ref PEER_DECODINGS: Arc<Mutex<HashMap<i32, SupportedDecoding>>> = Default::default();
+ static ref CODEC_NAME: Arc<Mutex<CodecName>> = Arc::new(Mutex::new(CodecName::VP9));
}
-const SCORE_VPX: i32 = 90;
#[derive(Debug, Clone)]
pub struct HwEncoderConfig {
- pub codec_name: String,
+ pub name: String,
pub width: usize,
pub height: usize,
pub bitrate: i32,
@@ -58,10 +58,6 @@ pub trait EncoderApi {
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>;
}
-pub struct DecoderCfg {
- pub vpx: VpxDecoderConfig,
-}
-
pub struct Encoder {
pub codec: Box<dyn EncoderApi>,
}
@@ -81,7 +77,8 @@ impl DerefMut for Encoder {
}
pub struct Decoder {
- vpx: VpxDecoder,
+ vp8: VpxDecoder,
+ vp9: VpxDecoder,
#[cfg(feature = "hwcodec")]
hw: HwDecoders,
#[cfg(feature = "hwcodec")]
@@ -91,10 +88,10 @@ pub struct Decoder {
}
#[derive(Debug, Clone)]
-pub enum EncoderUpdate {
- State(VideoCodecState),
+pub enum EncodingUpdate {
+ New(SupportedDecoding),
Remove,
- DisableHwIfNotExist,
+ NewOnlyVP9,
}
impl Encoder {
@@ -120,172 +117,156 @@ impl Encoder {
}
}
- // TODO
- pub fn update_video_encoder(id: i32, update: EncoderUpdate) {
+ pub fn update(id: i32, update: EncodingUpdate) {
+ let mut decodings = PEER_DECODINGS.lock().unwrap();
+ match update {
+ EncodingUpdate::New(decoding) => {
+ decodings.insert(id, decoding);
+ }
+ EncodingUpdate::Remove => {
+ decodings.remove(&id);
+ }
+ EncodingUpdate::NewOnlyVP9 => {
+ decodings.insert(
+ id,
+ SupportedDecoding {
+ ability_vp9: 1,
+ ..Default::default()
+ },
+ );
+ }
+ }
+
+ let vp8_useable = decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_vp8 > 0);
+ #[allow(unused_mut)]
+ let mut h264_name = None;
+ #[allow(unused_mut)]
+ let mut h265_name = None;
#[cfg(feature = "hwcodec")]
{
- let mut states = PEER_DECODER_STATES.lock().unwrap();
- match update {
- EncoderUpdate::State(state) => {
- states.insert(id, state);
- }
- EncoderUpdate::Remove => {
- states.remove(&id);
- }
- EncoderUpdate::DisableHwIfNotExist => {
- if !states.contains_key(&id) {
- states.insert(id, VideoCodecState::default());
- }
- }
+ let best = HwEncoder::best();
+ let h264_useable =
+ decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_h264 > 0);
+ let h265_useable =
+ decodings.len() > 0 && decodings.iter().all(|(_, s)| s.ability_h265 > 0);
+ if h264_useable {
+ h264_name = best.h264.map_or(None, |c| Some(c.name));
}
- let name = HwEncoder::current_name();
- if states.len() > 0 {
- let best = HwEncoder::best();
- let enabled_h264 = best.h264.is_some()
- && states.len() > 0
- && states.iter().all(|(_, s)| s.score_h264 > 0);
- let enabled_h265 = best.h265.is_some()
- && states.len() > 0
- && states.iter().all(|(_, s)| s.score_h265 > 0);
-
- // Preference first
- let mut preference = PreferCodec::Auto;
- let preferences: Vec<_> = states
- .iter()
- .filter(|(_, s)| {
- s.prefer == PreferCodec::VPX.into()
- || s.prefer == PreferCodec::H264.into() && enabled_h264
- || s.prefer == PreferCodec::H265.into() && enabled_h265
- })
- .map(|(_, s)| s.prefer)
- .collect();
- if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) {
- preference = preferences[0].enum_value_or(PreferCodec::Auto);
- }
-
- match preference {
- PreferCodec::VPX => *name.lock().unwrap() = None,
- PreferCodec::H264 => {
- *name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name))
- }
- PreferCodec::H265 => {
- *name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name))
- }
- PreferCodec::Auto => {
- // score encoder
- let mut score_vpx = SCORE_VPX;
- let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score);
- let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score);
-
- // score decoder
- score_vpx += states.iter().map(|s| s.1.score_vpx).sum::<i32>();
- if enabled_h264 {
- score_h264 += states.iter().map(|s| s.1.score_h264).sum::<i32>();
- }
- if enabled_h265 {
- score_h265 += states.iter().map(|s| s.1.score_h265).sum::<i32>();
- }
-
- if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 {
- *name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name));
- } else if enabled_h264
- && score_h264 >= score_vpx
- && score_h264 >= score_h265
- {
- *name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name));
- } else {
- *name.lock().unwrap() = None;
- }
- }
- }
-
- log::info!(
- "connection count:{}, used preference:{:?}, encoder:{:?}",
- states.len(),
- preference,
- name.lock().unwrap()
- )
- } else {
- *name.lock().unwrap() = None;
+ if h265_useable {
+ h265_name = best.h265.map_or(None, |c| Some(c.name));
}
}
- #[cfg(not(feature = "hwcodec"))]
- {
- let _ = id;
- let _ = update;
+
+ let mut name = CODEC_NAME.lock().unwrap();
+ let mut preference = PreferCodec::Auto;
+ let preferences: Vec<_> = decodings
+ .iter()
+ .filter(|(_, s)| {
+ s.prefer == PreferCodec::VP9.into()
+ || s.prefer == PreferCodec::VP8.into() && vp8_useable
+ || s.prefer == PreferCodec::H264.into() && h264_name.is_some()
+ || s.prefer == PreferCodec::H265.into() && h265_name.is_some()
+ })
+ .map(|(_, s)| s.prefer)
+ .collect();
+ if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) {
+ preference = preferences[0].enum_value_or(PreferCodec::Auto);
}
+
+ #[allow(unused_mut)]
+ let mut auto_codec = CodecName::VP9;
+ #[cfg(not(any(target_os = "android", target_os = "ios")))]
+ if vp8_useable && System::new_all().total_memory() <= 4 * 1024 * 1024 * 1024 {
+ // 4 Gb
+ auto_codec = CodecName::VP8
+ }
+
+ match preference {
+ PreferCodec::VP8 => *name = CodecName::VP8,
+ PreferCodec::VP9 => *name = CodecName::VP9,
+ PreferCodec::H264 => *name = h264_name.map_or(auto_codec, |c| CodecName::H264(c)),
+ PreferCodec::H265 => *name = h265_name.map_or(auto_codec, |c| CodecName::H265(c)),
+ PreferCodec::Auto => *name = auto_codec,
+ }
+
+ log::info!(
+ "connection count:{}, used preference:{:?}, encoder:{:?}",
+ decodings.len(),
+ preference,
+ *name
+ )
}
+
#[inline]
- pub fn current_hw_encoder_name() -> Option<String> {
- #[cfg(feature = "hwcodec")]
- if enable_hwcodec_option() {
- return HwEncoder::current_name().lock().unwrap().clone();
- } else {
- return None;
- }
- #[cfg(not(feature = "hwcodec"))]
- return None;
+ pub fn negotiated_codec() -> CodecName {
+ CODEC_NAME.lock().unwrap().clone()
}
- pub fn supported_encoding() -> (bool, bool) {
+ pub fn supported_encoding() -> SupportedEncoding {
+ #[allow(unused_mut)]
+ let mut encoding = SupportedEncoding {
+ vp8: true,
+ ..Default::default()
+ };
#[cfg(feature = "hwcodec")]
if enable_hwcodec_option() {
let best = HwEncoder::best();
- (
- best.h264.as_ref().map_or(false, |c| c.score > 0),
- best.h265.as_ref().map_or(false, |c| c.score > 0),
- )
- } else {
- (false, false)
+ encoding.h264 = best.h264.is_some();
+ encoding.h265 = best.h265.is_some();
}
- #[cfg(not(feature = "hwcodec"))]
- (false, false)
+ encoding
}
}
impl Decoder {
- pub fn video_codec_state(_id: &str) -> VideoCodecState {
+ pub fn supported_decodings(id_for_perfer: Option<&str>) -> SupportedDecoding {
+ #[allow(unused_mut)]
+ let mut decoding = SupportedDecoding {
+ ability_vp8: 1,
+ ability_vp9: 1,
+ prefer: id_for_perfer
+ .map_or(PreferCodec::Auto, |id| Self::codec_preference(id))
+ .into(),
+ ..Default::default()
+ };
#[cfg(feature = "hwcodec")]
if enable_hwcodec_option() {
let best = HwDecoder::best();
- return VideoCodecState {
- score_vpx: SCORE_VPX,
- score_h264: best.h264.map_or(0, |c| c.score),
- score_h265: best.h265.map_or(0, |c| c.score),
- prefer: Self::codec_preference(_id).into(),
- ..Default::default()
- };
+ decoding.ability_h264 = if best.h264.is_some() { 1 } else { 0 };
+ decoding.ability_h265 = if best.h265.is_some() { 1 } else { 0 };
}
#[cfg(feature = "mediacodec")]
if enable_hwcodec_option() {
- let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
- 92
- } else {
- 0
- };
- let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
- 94
- } else {
- 0
- };
- return VideoCodecState {
- score_vpx: SCORE_VPX,
- score_h264,
- score_h265,
- prefer: Self::codec_preference(_id).into(),
- ..Default::default()
- };
- }
- VideoCodecState {
- score_vpx: SCORE_VPX,
- ..Default::default()
+ decoding.ability_h264 =
+ if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
+ 1
+ } else {
+ 0
+ };
+ decoding.ability_h265 =
+ if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
+ 1
+ } else {
+ 0
+ };
}
+ decoding
}
- pub fn new(config: DecoderCfg) -> Decoder {
- let vpx = VpxDecoder::new(config.vpx).unwrap();
+ pub fn new() -> Decoder {
+ let vp8 = VpxDecoder::new(VpxDecoderConfig {
+ codec: VpxVideoCodecId::VP8,
+ num_threads: (num_cpus::get() / 2) as _,
+ })
+ .unwrap();
+ let vp9 = VpxDecoder::new(VpxDecoderConfig {
+ codec: VpxVideoCodecId::VP9,
+ num_threads: (num_cpus::get() / 2) as _,
+ })
+ .unwrap();
Decoder {
- vpx,
+ vp8,
+ vp9,
#[cfg(feature = "hwcodec")]
hw: if enable_hwcodec_option() {
HwDecoder::new_decoders()
@@ -310,8 +291,11 @@ impl Decoder {
rgb: &mut Vec<u8>,
) -> ResultType<bool> {
match frame {
+ video_frame::Union::Vp8s(vp8s) => {
+ Decoder::handle_vpxs_video_frame(&mut self.vp8, vp8s, fmt, rgb)
+ }
video_frame::Union::Vp9s(vp9s) => {
- Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, fmt, rgb)
+ Decoder::handle_vpxs_video_frame(&mut self.vp9, vp9s, fmt, rgb)
}
#[cfg(feature = "hwcodec")]
video_frame::Union::H264s(h264s) => {
@@ -349,15 +333,15 @@ impl Decoder {
}
}
- fn handle_vp9s_video_frame(
+ fn handle_vpxs_video_frame(
decoder: &mut VpxDecoder,
- vp9s: &EncodedVideoFrames,
+ vpxs: &EncodedVideoFrames,
fmt: (ImageFormat, usize),
rgb: &mut Vec<u8>,
) -> ResultType<bool> {
let mut last_frame = Image::new();
- for vp9 in vp9s.frames.iter() {
- for frame in decoder.decode(&vp9.data)? {
+ for vpx in vpxs.frames.iter() {
+ for frame in decoder.decode(&vpx.data)? {
drop(last_frame);
last_frame = frame;
}
@@ -408,14 +392,15 @@ impl Decoder {
return Ok(false);
}
- #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
fn codec_preference(id: &str) -> PreferCodec {
let codec = PeerConfig::load(id)
.options
.get("codec-preference")
.map_or("".to_owned(), |c| c.to_owned());
- if codec == "vp9" {
- PreferCodec::VPX
+ if codec == "vp8" {
+ PreferCodec::VP8
+ } else if codec == "vp9" {
+ PreferCodec::VP9
} else if codec == "h264" {
PreferCodec::H264
} else if codec == "h265" {
diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs
index 2c69774fb..f4de4bf84 100644
--- a/libs/scrap/src/common/hwcodec.rs
+++ b/libs/scrap/src/common/hwcodec.rs
@@ -7,7 +7,7 @@ use hbb_common::{
anyhow::{anyhow, Context},
bytes::Bytes,
config::HwCodecConfig,
- lazy_static, log,
+ log,
message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame},
ResultType,
};
@@ -19,11 +19,6 @@ use hwcodec::{
Quality::{self, *},
RateControl::{self, *},
};
-use std::sync::{Arc, Mutex};
-
-lazy_static::lazy_static! {
- static ref HW_ENCODER_NAME: Arc<Mutex<Option<String>>> = Default::default();
-}
const CFG_KEY_ENCODER: &str = "bestHwEncoders";
const CFG_KEY_DECODER: &str = "bestHwDecoders";
@@ -49,7 +44,7 @@ impl EncoderApi for HwEncoder {
match cfg {
EncoderCfg::HW(config) => {
let ctx = EncodeContext {
- name: config.codec_name.clone(),
+ name: config.name.clone(),
width: config.width as _,
height: config.height as _,
pixfmt: DEFAULT_PIXFMT,
@@ -60,12 +55,12 @@ impl EncoderApi for HwEncoder {
quality: DEFAULT_HW_QUALITY,
rc: DEFAULT_RC,
};
- let format = match Encoder::format_from_name(config.codec_name.clone()) {
+ let format = match Encoder::format_from_name(config.name.clone()) {
Ok(format) => format,
Err(_) => {
return Err(anyhow!(format!(
"failed to get format from name:{}",
- config.codec_name
+ config.name
)))
}
};
@@ -133,10 +128,6 @@ impl HwEncoder {
})
}
- pub fn current_name() -> Arc<Mutex<Option<String>>> {
- HW_ENCODER_NAME.clone()
- }
-
pub fn encode(&mut self, bgra: &[u8]) -> ResultType<Vec<EncodeFrame>> {
match self.pixfmt {
AVPixelFormat::AV_PIX_FMT_YUV420P => hw::hw_bgra_to_i420(
diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs
index 0ad158cca..26b946401 100644
--- a/libs/scrap/src/common/mod.rs
+++ b/libs/scrap/src/common/mod.rs
@@ -1,4 +1,5 @@
pub use self::vpxcodec::*;
+use hbb_common::message_proto::{video_frame, VideoFrame};
cfg_if! {
if #[cfg(quartz)] {
@@ -92,3 +93,55 @@ pub fn is_cursor_embedded() -> bool {
pub fn is_cursor_embedded() -> bool {
false
}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum CodecName {
+ VP8,
+ VP9,
+ H264(String),
+ H265(String),
+}
+
+#[derive(PartialEq, Debug, Clone)]
+pub enum CodecFormat {
+ VP8,
+ VP9,
+ H264,
+ H265,
+ Unknown,
+}
+
+impl From<&VideoFrame> for CodecFormat {
+ fn from(it: &VideoFrame) -> Self {
+ match it.union {
+ Some(video_frame::Union::Vp8s(_)) => CodecFormat::VP8,
+ Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9,
+ Some(video_frame::Union::H264s(_)) => CodecFormat::H264,
+ Some(video_frame::Union::H265s(_)) => CodecFormat::H265,
+ _ => CodecFormat::Unknown,
+ }
+ }
+}
+
+impl From<&CodecName> for CodecFormat {
+ fn from(value: &CodecName) -> Self {
+ match value {
+ CodecName::VP8 => Self::VP8,
+ CodecName::VP9 => Self::VP9,
+ CodecName::H264(_) => Self::H264,
+ CodecName::H265(_) => Self::H265,
+ }
+ }
+}
+
+impl ToString for CodecFormat {
+ fn to_string(&self) -> String {
+ match self {
+ CodecFormat::VP8 => "VP8".into(),
+ CodecFormat::VP9 => "VP9".into(),
+ CodecFormat::H264 => "H264".into(),
+ CodecFormat::H265 => "H265".into(),
+ CodecFormat::Unknown => "Unknow".into(),
+ }
+ }
+}
diff --git a/libs/scrap/src/common/record.rs b/libs/scrap/src/common/record.rs
index 9f38f2d6a..9de70ae14 100644
--- a/libs/scrap/src/common/record.rs
+++ b/libs/scrap/src/common/record.rs
@@ -1,3 +1,4 @@
+use crate::CodecFormat;
#[cfg(feature = "hwcodec")]
use hbb_common::anyhow::anyhow;
use hbb_common::{
@@ -21,13 +22,6 @@ use webm::mux::{self, Segment, Track, VideoTrack, Writer};
const MIN_SECS: u64 = 1;
-#[derive(Debug, Clone, PartialEq)]
-pub enum RecordCodecID {
- VP9,
- H264,
- H265,
-}
-
#[derive(Debug, Clone)]
pub struct RecorderContext {
pub server: bool,
@@ -36,7 +30,7 @@ pub struct RecorderContext {
pub filename: String,
pub width: usize,
pub height: usize,
- pub codec_id: RecordCodecID,
+ pub format: CodecFormat,
pub tx: Option<Sender<RecordState>>,
}
@@ -55,8 +49,9 @@ impl RecorderContext {
}
let file = if self.server { "s" } else { "c" }.to_string()
+ &self.id.clone()
- + &chrono::Local::now().format("_%Y%m%d%H%M%S").to_string()
- + if self.codec_id == RecordCodecID::VP9 {
+ + &chrono::Local::now().format("_%Y%m%d%H%M%S_").to_string()
+ + &self.format.to_string()
+ + if self.format == CodecFormat::VP9 || self.format == CodecFormat::VP8 {
".webm"
} else {
".mp4"
@@ -107,8 +102,8 @@ impl DerefMut for Recorder {
impl Recorder {
pub fn new(mut ctx: RecorderContext) -> ResultType<Self> {
ctx.set_filename()?;
- let recorder = match ctx.codec_id {
- RecordCodecID::VP9 => Recorder {
+ let recorder = match ctx.format {
+ CodecFormat::VP8 | CodecFormat::VP9 => Recorder {
inner: Box::new(WebmRecorder::new(ctx.clone())?),
ctx,
},
@@ -126,8 +121,8 @@ impl Recorder {
fn change(&mut self, mut ctx: RecorderContext) -> ResultType<()> {
ctx.set_filename()?;
- self.inner = match ctx.codec_id {
- RecordCodecID::VP9 => Box::new(WebmRecorder::new(ctx.clone())?),
+ self.inner = match ctx.format {
+ CodecFormat::VP8 | CodecFormat::VP9 => Box::new(WebmRecorder::new(ctx.clone())?),
#[cfg(feature = "hwcodec")]
_ => Box::new(HwRecorder::new(ctx.clone())?),
#[cfg(not(feature = "hwcodec"))]
@@ -148,10 +143,19 @@ impl Recorder {
pub fn write_frame(&mut self, frame: &video_frame::Union) -> ResultType<()> {
match frame {
- video_frame::Union::Vp9s(vp9s) => {
- if self.ctx.codec_id != RecordCodecID::VP9 {
+ video_frame::Union::Vp8s(vp8s) => {
+ if self.ctx.format != CodecFormat::VP8 {
self.change(RecorderContext {
- codec_id: RecordCodecID::VP9,
+ format: CodecFormat::VP8,
+ ..self.ctx.clone()
+ })?;
+ }
+ vp8s.frames.iter().map(|f| self.write_video(f)).count();
+ }
+ video_frame::Union::Vp9s(vp9s) => {
+ if self.ctx.format != CodecFormat::VP9 {
+ self.change(RecorderContext {
+ format: CodecFormat::VP9,
..self.ctx.clone()
})?;
}
@@ -159,25 +163,25 @@ impl Recorder {
}
#[cfg(feature = "hwcodec")]
video_frame::Union::H264s(h264s) => {
- if self.ctx.codec_id != RecordCodecID::H264 {
+ if self.ctx.format != CodecFormat::H264 {
self.change(RecorderContext {
- codec_id: RecordCodecID::H264,
+ format: CodecFormat::H264,
..self.ctx.clone()
})?;
}
- if self.ctx.codec_id == RecordCodecID::H264 {
+ if self.ctx.format == CodecFormat::H264 {
h264s.frames.iter().map(|f| self.write_video(f)).count();
}
}
#[cfg(feature = "hwcodec")]
video_frame::Union::H265s(h265s) => {
- if self.ctx.codec_id != RecordCodecID::H265 {
+ if self.ctx.format != CodecFormat::H265 {
self.change(RecorderContext {
- codec_id: RecordCodecID::H265,
+ format: CodecFormat::H265,
..self.ctx.clone()
})?;
}
- if self.ctx.codec_id == RecordCodecID::H265 {
+ if self.ctx.format == CodecFormat::H265 {
h265s.frames.iter().map(|f| self.write_video(f)).count();
}
}
@@ -221,7 +225,11 @@ impl RecorderApi for WebmRecorder {
ctx.width as _,
ctx.height as _,
None,
- mux::VideoCodecId::VP9,
+ if ctx.format == CodecFormat::VP9 {
+ mux::VideoCodecId::VP9
+ } else {
+ mux::VideoCodecId::VP8
+ },
);
Ok(WebmRecorder {
vt,
@@ -279,7 +287,7 @@ impl RecorderApi for HwRecorder {
filename: ctx.filename.clone(),
width: ctx.width,
height: ctx.height,
- is265: ctx.codec_id == RecordCodecID::H265,
+ is265: ctx.format == CodecFormat::H265,
framerate: crate::hwcodec::DEFAULT_TIME_BASE[1] as _,
})
.map_err(|_| anyhow!("Failed to create hardware muxer"))?;
diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs
index 3df9c0461..4820ea171 100644
--- a/libs/scrap/src/common/vpxcodec.rs
+++ b/libs/scrap/src/common/vpxcodec.rs
@@ -30,6 +30,7 @@ pub struct VpxEncoder {
ctx: vpx_codec_ctx_t,
width: usize,
height: usize,
+ id: VpxVideoCodecId,
}
pub struct VpxDecoder {
@@ -97,15 +98,10 @@ impl EncoderApi for VpxEncoder {
{
match cfg {
crate::codec::EncoderCfg::VPX(config) => {
- let i;
- if cfg!(feature = "VP8") {
- i = match config.codec {
- VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()),
- VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()),
- };
- } else {
- i = call_vpx_ptr!(vpx_codec_vp9_cx());
- }
+ let i = match config.codec {
+ VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()),
+ VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()),
+ };
let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0));
@@ -187,12 +183,17 @@ impl EncoderApi for VpxEncoder {
VP9E_SET_TILE_COLUMNS as _,
4 as c_int
));
+ } else if config.codec == VpxVideoCodecId::VP8 {
+ // https://github.com/webmproject/libvpx/blob/972149cafeb71d6f08df89e91a0130d6a38c4b15/vpx/vp8cx.h#L172
+ // https://groups.google.com/a/webmproject.org/g/webm-discuss/c/DJhSrmfQ61M
+ call_vpx!(vpx_codec_control_(&mut ctx, VP8E_SET_CPUUSED as _, 12,));
}
Ok(Self {
ctx,
width: config.width as _,
height: config.height as _,
+ id: config.codec,
})
}
_ => Err(anyhow!("encoder type mismatch")),
@@ -213,7 +214,7 @@ impl EncoderApi for VpxEncoder {
// to-do: flush periodically, e.g. 1 second
if frames.len() > 0 {
- Ok(VpxEncoder::create_msg(frames))
+ Ok(VpxEncoder::create_msg(self.id, frames))
} else {
Err(anyhow!("no valid frame"))
}
@@ -280,13 +281,17 @@ impl VpxEncoder {
}
#[inline]
- fn create_msg(vp9s: Vec<EncodedVideoFrame>) -> Message {
+ pub fn create_msg(codec_id: VpxVideoCodecId, frames: Vec<EncodedVideoFrame>) -> Message {
let mut msg_out = Message::new();
let mut vf = VideoFrame::new();
- vf.set_vp9s(EncodedVideoFrames {
- frames: vp9s.into(),
+ let vpxs = EncodedVideoFrames {
+ frames: frames.into(),
..Default::default()
- });
+ };
+ match codec_id {
+ VpxVideoCodecId::VP8 => vf.set_vp8s(vpxs),
+ VpxVideoCodecId::VP9 => vf.set_vp9s(vpxs),
+ }
msg_out.set_video_frame(vf);
msg_out
}
@@ -382,15 +387,10 @@ impl VpxDecoder {
pub fn new(config: VpxDecoderConfig) -> Result<Self> {
// This is sound because `vpx_codec_ctx` is a repr(C) struct without any field that can
// cause UB if uninitialized.
- let i;
- if cfg!(feature = "VP8") {
- i = match config.codec {
- VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()),
- VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()),
- };
- } else {
- i = call_vpx_ptr!(vpx_codec_vp9_dx());
- }
+ let i = match config.codec {
+ VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_dx()),
+ VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_dx()),
+ };
let mut ctx = Default::default();
let cfg = vpx_codec_dec_cfg_t {
threads: if config.num_threads == 0 {
diff --git a/src/client.rs b/src/client.rs
index 2f745b70c..ae93ccfcf 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -44,9 +44,9 @@ use hbb_common::{
};
pub use helper::*;
use scrap::{
- codec::{Decoder, DecoderCfg},
+ codec::Decoder,
record::{Recorder, RecorderContext},
- ImageFormat, VpxDecoderConfig, VpxVideoCodecId,
+ ImageFormat,
};
use crate::{
@@ -917,12 +917,7 @@ impl VideoHandler {
/// Create a new video handler.
pub fn new() -> Self {
VideoHandler {
- decoder: Decoder::new(DecoderCfg {
- vpx: VpxDecoderConfig {
- codec: VpxVideoCodecId::VP9,
- num_threads: (num_cpus::get() / 2) as _,
- },
- }),
+ decoder: Decoder::new(),
rgb: Default::default(),
recorder: Default::default(),
record: false,
@@ -954,12 +949,7 @@ impl VideoHandler {
/// Reset the decoder.
pub fn reset(&mut self) {
- self.decoder = Decoder::new(DecoderCfg {
- vpx: VpxDecoderConfig {
- codec: VpxVideoCodecId::VP9,
- num_threads: 1,
- },
- });
+ self.decoder = Decoder::new();
}
/// Start or stop screen record.
@@ -973,7 +963,7 @@ impl VideoHandler {
filename: "".to_owned(),
width: w as _,
height: h as _,
- codec_id: scrap::record::RecordCodecID::VP9,
+ format: scrap::CodecFormat::VP9,
tx: None,
})
.map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r))));
@@ -999,7 +989,7 @@ pub struct LoginConfigHandler {
pub conn_id: i32,
features: Option<Features>,
session_id: u64,
- pub supported_encoding: Option<(bool, bool)>,
+ pub supported_encoding: SupportedEncoding,
pub restarting_remote_device: bool,
pub force_relay: bool,
pub direct: Option<bool>,
@@ -1047,7 +1037,7 @@ impl LoginConfigHandler {
self.remember = !config.password.is_empty();
self.config = config;
self.session_id = rand::random();
- self.supported_encoding = None;
+ self.supported_encoding = Default::default();
self.restarting_remote_device = false;
self.force_relay = !self.get_option("force-always-relay").is_empty() || force_relay;
self.direct = None;
@@ -1331,8 +1321,8 @@ impl LoginConfigHandler {
msg.disable_clipboard = BoolOption::Yes.into();
n += 1;
}
- let state = Decoder::video_codec_state(&self.id);
- msg.video_codec_state = hbb_common::protobuf::MessageField::some(state);
+ msg.supported_decoding =
+ hbb_common::protobuf::MessageField::some(Decoder::supported_decodings(Some(&self.id)));
n += 1;
if n > 0 {
@@ -1565,10 +1555,7 @@ impl LoginConfigHandler {
self.conn_id = pi.conn_id;
// no matter if change, for update file time
self.save_config(config);
- #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
- {
- self.supported_encoding = Some((pi.encoding.h264, pi.encoding.h265));
- }
+ self.supported_encoding = pi.encoding.clone().unwrap_or_default();
}
pub fn get_remote_dir(&self) -> String {
@@ -1626,10 +1613,10 @@ impl LoginConfigHandler {
}
pub fn change_prefer_codec(&self) -> Message {
- let state = scrap::codec::Decoder::video_codec_state(&self.id);
+ let decoding = scrap::codec::Decoder::supported_decodings(Some(&self.id));
let mut misc = Misc::new();
misc.set_option(OptionMessage {
- video_codec_state: hbb_common::protobuf::MessageField::some(state),
+ supported_decoding: hbb_common::protobuf::MessageField::some(decoding),
..Default::default()
});
let mut msg_out = Message::new();
diff --git a/src/client/helper.rs b/src/client/helper.rs
index a9696a8e8..61844d908 100644
--- a/src/client/helper.rs
+++ b/src/client/helper.rs
@@ -1,37 +1,8 @@
use hbb_common::{
get_time,
- message_proto::{video_frame, Message, VideoFrame, VoiceCallRequest, VoiceCallResponse},
+ message_proto::{Message, VoiceCallRequest, VoiceCallResponse},
};
-
-#[derive(PartialEq, Debug, Clone)]
-pub enum CodecFormat {
- VP9,
- H264,
- H265,
- Unknown,
-}
-
-impl From<&VideoFrame> for CodecFormat {
- fn from(it: &VideoFrame) -> Self {
- match it.union {
- Some(video_frame::Union::Vp9s(_)) => CodecFormat::VP9,
- Some(video_frame::Union::H264s(_)) => CodecFormat::H264,
- Some(video_frame::Union::H265s(_)) => CodecFormat::H265,
- _ => CodecFormat::Unknown,
- }
- }
-}
-
-impl ToString for CodecFormat {
- fn to_string(&self) -> String {
- match self {
- CodecFormat::VP9 => "VP9".into(),
- CodecFormat::H264 => "H264".into(),
- CodecFormat::H265 => "H265".into(),
- CodecFormat::Unknown => "Unknow".into(),
- }
- }
-}
+use scrap::CodecFormat;
#[derive(Debug, Default)]
pub struct QualityStatus {
diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs
index b0bddc82e..0f662e015 100644
--- a/src/client/io_loop.rs
+++ b/src/client/io_loop.rs
@@ -29,10 +29,10 @@ use hbb_common::tokio::{
time::{self, Duration, Instant, Interval},
};
use hbb_common::{allow_err, fs, get_time, log, message_proto::*, Stream};
+use scrap::CodecFormat;
use crate::client::{
- new_voice_call_request, Client, CodecFormat, MediaData, MediaSender, QualityStatus, MILLI1,
- SEC30,
+ new_voice_call_request, Client, MediaData, MediaSender, QualityStatus, MILLI1, SEC30,
};
#[cfg(not(any(target_os = "android", target_os = "ios")))]
use crate::common::{self, update_clipboard};
@@ -817,11 +817,10 @@ impl<T: InvokeUiSession> Remote<T> {
}
fn contains_key_frame(vf: &VideoFrame) -> bool {
+ use video_frame::Union::*;
match &vf.union {
Some(vf) => match vf {
- video_frame::Union::Vp9s(f) => f.frames.iter().any(|e| e.key),
- video_frame::Union::H264s(f) => f.frames.iter().any(|e| e.key),
- video_frame::Union::H265s(f) => f.frames.iter().any(|e| e.key),
+ Vp8s(f) | Vp9s(f) | H264s(f) | H265s(f) => f.frames.iter().any(|e| e.key),
_ => false,
},
None => false,
diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs
index aee486b94..1d965ebc5 100644
--- a/src/flutter_ffi.rs
+++ b/src/flutter_ffi.rs
@@ -969,6 +969,13 @@ pub fn main_has_hwcodec() -> SyncReturn<bool> {
SyncReturn(has_hwcodec())
}
+pub fn main_supported_hwdecodings() -> SyncReturn<String> {
+ let decoding = supported_hwdecodings();
+ let msg = HashMap::from([("h264", decoding.0), ("h265", decoding.1)]);
+
+ SyncReturn(serde_json::ser::to_string(&msg).unwrap_or("".to_owned()))
+}
+
pub fn main_is_root() -> bool {
is_root()
}
@@ -1054,10 +1061,10 @@ pub fn session_send_note(id: String, note: String) {
}
}
-pub fn session_supported_hwcodec(id: String) -> String {
+pub fn session_alternative_codecs(id: String) -> String {
if let Some(session) = SESSIONS.read().unwrap().get(&id) {
- let (h264, h265) = session.supported_hwcodec();
- let msg = HashMap::from([("h264", h264), ("h265", h265)]);
+ let (vp8, h264, h265) = session.alternative_codecs();
+ let msg = HashMap::from([("vp8", vp8), ("h264", h264), ("h265", h265)]);
serde_json::ser::to_string(&msg).unwrap_or("".to_owned())
} else {
String::new()
diff --git a/src/server/connection.rs b/src/server/connection.rs
index 725ff43ee..539d96ddd 100644
--- a/src/server/connection.rs
+++ b/src/server/connection.rs
@@ -533,7 +533,7 @@ impl Connection {
let _ = privacy_mode::turn_off_privacy(0);
}
video_service::notify_video_frame_fetched(id, None);
- scrap::codec::Encoder::update_video_encoder(id, scrap::codec::EncoderUpdate::Remove);
+ scrap::codec::Encoder::update(id, scrap::codec::EncodingUpdate::Remove);
video_service::VIDEO_QOS.lock().unwrap().reset();
if conn.authorized {
password::update_temporary_password();
@@ -860,16 +860,7 @@ impl Connection {
}
}
- #[cfg(feature = "hwcodec")]
- {
- let (h264, h265) = scrap::codec::Encoder::supported_encoding();
- pi.encoding = Some(SupportedEncoding {
- h264,
- h265,
- ..Default::default()
- })
- .into();
- }
+ pi.encoding = Some(scrap::codec::Encoder::supported_encoding()).into();
if self.port_forward_socket.is_some() {
let mut msg_out = Message::new();
@@ -1145,21 +1136,21 @@ impl Connection {
self.lr = lr.clone();
if let Some(o) = lr.option.as_ref() {
self.options_in_login = Some(o.clone());
- if let Some(q) = o.video_codec_state.clone().take() {
- scrap::codec::Encoder::update_video_encoder(
+ if let Some(q) = o.supported_decoding.clone().take() {
+ scrap::codec::Encoder::update(
self.inner.id(),
- scrap::codec::EncoderUpdate::State(q),
+ scrap::codec::EncodingUpdate::New(q),
);
} else {
- scrap::codec::Encoder::update_video_encoder(
+ scrap::codec::Encoder::update(
self.inner.id(),
- scrap::codec::EncoderUpdate::DisableHwIfNotExist,
+ scrap::codec::EncodingUpdate::NewOnlyVP9,
);
}
} else {
- scrap::codec::Encoder::update_video_encoder(
+ scrap::codec::Encoder::update(
self.inner.id(),
- scrap::codec::EncoderUpdate::DisableHwIfNotExist,
+ scrap::codec::EncodingUpdate::NewOnlyVP9,
);
}
self.video_ack_required = lr.video_ack_required;
@@ -1758,11 +1749,8 @@ impl Connection {
.unwrap()
.update_user_fps(o.custom_fps as _);
}
- if let Some(q) = o.video_codec_state.clone().take() {
- scrap::codec::Encoder::update_video_encoder(
- self.inner.id(),
- scrap::codec::EncoderUpdate::State(q),
- );
+ if let Some(q) = o.supported_decoding.clone().take() {
+ scrap::codec::Encoder::update(self.inner.id(), scrap::codec::EncodingUpdate::New(q));
}
if let Ok(q) = o.lock_after_session_end.enum_value() {
if q != BoolOption::NotSet {
diff --git a/src/server/video_service.rs b/src/server/video_service.rs
index 205d0584c..691ca4abe 100644
--- a/src/server/video_service.rs
+++ b/src/server/video_service.rs
@@ -31,7 +31,7 @@ use scrap::{
codec::{Encoder, EncoderCfg, HwEncoderConfig},
record::{Recorder, RecorderContext},
vpxcodec::{VpxEncoderConfig, VpxVideoCodecId},
- Display, TraitCapturer,
+ CodecName, Display, TraitCapturer,
};
#[cfg(windows)]
use std::sync::Once;
@@ -468,21 +468,29 @@ fn run(sp: GenericService) -> ResultType<()> {
drop(video_qos);
log::info!("init bitrate={}, abr enabled:{}", bitrate, abr);
- let encoder_cfg = match Encoder::current_hw_encoder_name() {
- Some(codec_name) => EncoderCfg::HW(HwEncoderConfig {
- codec_name,
- width: c.width,
- height: c.height,
- bitrate: bitrate as _,
- }),
- None => EncoderCfg::VPX(VpxEncoderConfig {
- width: c.width as _,
- height: c.height as _,
- timebase: [1, 1000], // Output timestamp precision
- bitrate,
- codec: VpxVideoCodecId::VP9,
- num_threads: (num_cpus::get() / 2) as _,
- }),
+ let encoder_cfg = match Encoder::negotiated_codec() {
+ scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
+ EncoderCfg::HW(HwEncoderConfig {
+ name,
+ width: c.width,
+ height: c.height,
+ bitrate: bitrate as _,
+ })
+ }
+ name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
+ EncoderCfg::VPX(VpxEncoderConfig {
+ width: c.width as _,
+ height: c.height as _,
+ timebase: [1, 1000], // Output timestamp precision
+ bitrate,
+ codec: if name == scrap::CodecName::VP8 {
+ VpxVideoCodecId::VP8
+ } else {
+ VpxVideoCodecId::VP9
+ },
+ num_threads: (num_cpus::get() / 2) as _,
+ })
+ }
};
let mut encoder;
@@ -526,7 +534,7 @@ fn run(sp: GenericService) -> ResultType<()> {
let mut try_gdi = 1;
#[cfg(windows)]
log::info!("gdi: {}", c.is_gdi());
- let codec_name = Encoder::current_hw_encoder_name();
+ let codec_name = Encoder::negotiated_codec();
let recorder = get_recorder(c.width, c.height, &codec_name);
#[cfg(windows)]
start_uac_elevation_check();
@@ -557,7 +565,7 @@ fn run(sp: GenericService) -> ResultType<()> {
*SWITCH.lock().unwrap() = true;
bail!("SWITCH");
}
- if codec_name != Encoder::current_hw_encoder_name() {
+ if codec_name != Encoder::negotiated_codec() {
bail!("SWITCH");
}
#[cfg(windows)]
@@ -603,8 +611,14 @@ fn run(sp: GenericService) -> ResultType<()> {
let time = now - start;
let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
match frame {
+ scrap::Frame::VP8(data) => {
+ let send_conn_ids =
+ handle_one_frame_encoded(VpxVideoCodecId::VP8, &sp, data, ms)?;
+ frame_controller.set_send(now, send_conn_ids);
+ }
scrap::Frame::VP9(data) => {
- let send_conn_ids = handle_one_frame_encoded(&sp, data, ms)?;
+ let send_conn_ids =
+ handle_one_frame_encoded(VpxVideoCodecId::VP9, &sp, data, ms)?;
frame_controller.set_send(now, send_conn_ids);
}
scrap::Frame::RAW(data) => {
@@ -717,12 +731,11 @@ fn run(sp: GenericService) -> ResultType<()> {
fn get_recorder(
width: usize,
height: usize,
- codec_name: &Option<String>,
+ codec_name: &CodecName,
) -> Arc<Mutex<Option<Recorder>>> {
#[cfg(not(target_os = "ios"))]
let recorder = if !Config::get_option("allow-auto-record-incoming").is_empty() {
use crate::hbbs_http::record_upload;
- use scrap::record::RecordCodecID::*;
let tx = if record_upload::is_enable() {
let (tx, rx) = std::sync::mpsc::channel();
@@ -731,16 +744,6 @@ fn get_recorder(
} else {
None
};
- let codec_id = match codec_name {
- Some(name) => {
- if name.contains("264") {
- H264
- } else {
- H265
- }
- }
- None => VP9,
- };
Recorder::new(RecorderContext {
server: true,
id: Config::get_id(),
@@ -748,7 +751,7 @@ fn get_recorder(
filename: "".to_owned(),
width,
height,
- codec_id,
+ format: codec_name.into(),
tx,
})
.map_or(Default::default(), |r| Arc::new(Mutex::new(Some(r))))
@@ -775,19 +778,6 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu
Ok(())
}
-#[inline]
-#[cfg(any(target_os = "android", target_os = "ios"))]
-fn create_msg(vp9s: Vec<EncodedVideoFrame>) -> Message {
- let mut msg_out = Message::new();
- let mut vf = VideoFrame::new();
- vf.set_vp9s(EncodedVideoFrames {
- frames: vp9s.into(),
- ..Default::default()
- });
- msg_out.set_video_frame(vf);
- msg_out
-}
-
#[inline]
fn handle_one_frame(
sp: &GenericService,
@@ -820,6 +810,7 @@ fn handle_one_frame(
#[inline]
#[cfg(any(target_os = "android", target_os = "ios"))]
pub fn handle_one_frame_encoded(
+ codec: VpxVideoCodecId,
sp: &GenericService,
frame: &[u8],
ms: i64,
@@ -831,13 +822,13 @@ pub fn handle_one_frame_encoded(
}
Ok(())
})?;
- let vp9_frame = EncodedVideoFrame {
+ let vpx_frame = EncodedVideoFrame {
data: frame.to_vec().into(),
key: true,
pts: ms,
..Default::default()
};
- let send_conn_ids = sp.send_video_frame(create_msg(vec![vp9_frame]));
+ let send_conn_ids = sp.send_video_frame(scrap::VpxEncoder::create_msg(codec, vec![vpx_frame]));
Ok(send_conn_ids)
}
diff --git a/src/ui/header.tis b/src/ui/header.tis
index 257ba417e..af4f1e349 100644
--- a/src/ui/header.tis
+++ b/src/ui/header.tis
@@ -161,8 +161,8 @@ class Header: Reactor.Component {
}
function renderDisplayPop() {
- var codecs = handler.supported_hwcodec();
- var show_codec = handler.has_hwcodec() && (codecs[0] || codecs[1]);
+ var codecs = handler.alternative_codecs();
+ var show_codec = codecs[0] || codecs[1] || codecs[2];
var cursor_embedded = false;
if ((pi.displays || []).length > 0) {
@@ -186,9 +186,10 @@ class Header: Reactor.Component {
{show_codec ? <div>
<div .separator />
<li #auto type="codec-preference"><span>{svg_checkmark}</span>Auto</li>
+ {codecs[0] ? <li #vp8 type="codec-preference"><span>{svg_checkmark}</span>VP8</li> : ""}
<li #vp9 type="codec-preference"><span>{svg_checkmark}</span>VP9</li>
- {codecs[0] ? <li #h264 type="codec-preference"><span>{svg_checkmark}</span>H264</li> : ""}
- {codecs[1] ? <li #h265 type="codec-preference"><span>{svg_checkmark}</span>H265</li> : ""}
+ {codecs[1] ? <li #h264 type="codec-preference"><span>{svg_checkmark}</span>H264</li> : ""}
+ {codecs[2] ? <li #h265 type="codec-preference"><span>{svg_checkmark}</span>H265</li> : ""}
</div> : ""}
<div .separator />
{!cursor_embedded && <li #show-remote-cursor .toggle-option><span>{svg_checkmark}</span>{translate('Show remote cursor')}</li>}
diff --git a/src/ui/remote.rs b/src/ui/remote.rs
index 68decf955..fc878cf1f 100644
--- a/src/ui/remote.rs
+++ b/src/ui/remote.rs
@@ -20,7 +20,6 @@ use hbb_common::{
use crate::{
client::*,
- ui_interface::has_hwcodec,
ui_session_interface::{InvokeUiSession, Session},
};
@@ -197,7 +196,14 @@ impl InvokeUiSession for SciterHandler {
self.call("confirmDeleteFiles", &make_args!(id, i, name));
}
- fn override_file_confirm(&self, id: i32, file_num: i32, to: String, is_upload: bool, is_identical: bool) {
+ fn override_file_confirm(
+ &self,
+ id: i32,
+ file_num: i32,
+ to: String,
+ is_upload: bool,
+ is_identical: bool,
+ ) {
self.call(
"overrideFileConfirm",
&make_args!(id, file_num, to, is_upload, is_identical),
@@ -451,8 +457,7 @@ impl sciter::EventHandler for SciterSession {
fn set_write_override(i32, i32, bool, bool, bool);
fn get_keyboard_mode();
fn save_keyboard_mode(String);
- fn has_hwcodec();
- fn supported_hwcodec();
+ fn alternative_codecs();
fn change_prefer_codec();
fn restart_remote_device();
fn request_voice_call();
@@ -504,10 +509,6 @@ impl SciterSession {
v
}
- fn has_hwcodec(&self) -> bool {
- has_hwcodec()
- }
-
pub fn t(&self, name: String) -> String {
crate::client::translate(name)
}
@@ -516,9 +517,10 @@ impl SciterSession {
super::get_icon()
}
- fn supported_hwcodec(&self) -> Value {
- let (h264, h265) = self.0.supported_hwcodec();
+ fn alternative_codecs(&self) -> Value {
+ let (vp8, h264, h265) = self.0.alternative_codecs();
let mut v = Value::array(0);
+ v.push(vp8);
v.push(h264);
v.push(h265);
v
diff --git a/src/ui_interface.rs b/src/ui_interface.rs
index 11357be4a..26d470fe0 100644
--- a/src/ui_interface.rs
+++ b/src/ui_interface.rs
@@ -752,6 +752,13 @@ pub fn has_hwcodec() -> bool {
return true;
}
+#[cfg(feature = "flutter")]
+#[inline]
+pub fn supported_hwdecodings() -> (bool, bool) {
+ let decoding = scrap::codec::Decoder::supported_decodings(None);
+ (decoding.ability_h264 > 0, decoding.ability_h265 > 0)
+}
+
#[cfg(not(any(target_os = "android", target_os = "ios")))]
#[inline]
pub fn is_root() -> bool {
diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs
index f89be4879..3dee89a6e 100644
--- a/src/ui_session_interface.rs
+++ b/src/ui_session_interface.rs
@@ -216,24 +216,16 @@ impl<T: InvokeUiSession> Session<T> {
true
}
- pub fn supported_hwcodec(&self) -> (bool, bool) {
- #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
- {
- let decoder = scrap::codec::Decoder::video_codec_state(&self.id);
- let mut h264 = decoder.score_h264 > 0;
- let mut h265 = decoder.score_h265 > 0;
- let (encoding_264, encoding_265) = self
- .lc
- .read()
- .unwrap()
- .supported_encoding
- .unwrap_or_default();
- h264 = h264 && encoding_264;
- h265 = h265 && encoding_265;
- return (h264, h265);
- }
- #[allow(unreachable_code)]
- (false, false)
+ pub fn alternative_codecs(&self) -> (bool, bool, bool) {
+ let decoder = scrap::codec::Decoder::supported_decodings(None);
+ let mut vp8 = decoder.ability_vp8 > 0;
+ let mut h264 = decoder.ability_h264 > 0;
+ let mut h265 = decoder.ability_h265 > 0;
+ let enc = &self.lc.read().unwrap().supported_encoding;
+ vp8 = vp8 && enc.vp8;
+ h264 = h264 && enc.h264;
+ h265 = h265 && enc.h265;
+ (vp8, h264, h265)
}
pub fn change_prefer_codec(&self) {