From 5bd08bf0a783838d4028a09d9fc47ea02281d9f6 Mon Sep 17 00:00:00 2001 From: csf Date: Tue, 5 Jul 2022 22:17:34 +0800 Subject: [PATCH] client side handle codec format --- libs/hbb_common/protos/message.proto | 8 ---- libs/scrap/src/common/codec.rs | 4 +- libs/scrap/src/common/hwcodec.rs | 9 +--- libs/scrap/src/common/vpxcodec.rs | 14 +------ src/client.rs | 2 +- src/client/helper.rs | 32 +++++++++++++- src/server/connection.rs | 1 - src/server/video_service.rs | 9 +--- src/ui/remote.rs | 62 ++++++++++++---------------- src/ui/remote.tis | 8 ++-- 10 files changed, 68 insertions(+), 81 deletions(-) diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 40da75a5d..f6c57b1e4 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -479,18 +479,10 @@ message OptionMessage { } message TestDelay { - enum CodecFormat { - Unknown = 0; - VP8 = 1; - VP9 = 2; - H264 = 3; - H265 = 4; - } int64 time = 1; bool from_client = 2; uint32 last_delay = 3; uint32 target_bitrate = 4; - CodecFormat codec_format = 5; } message PublicKey { diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 2a05c7326..866a708a3 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -12,7 +12,7 @@ use crate::vpxcodec::*; use hbb_common::{ anyhow::anyhow, log, - message_proto::{test_delay, video_frame, Message, VP9s, VideoCodecState}, + message_proto::{video_frame, Message, VP9s, VideoCodecState}, ResultType, }; #[cfg(feature = "hwcodec")] @@ -53,8 +53,6 @@ pub trait EncoderApi { fn use_yuv(&self) -> bool; fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>; - - fn get_codec_format(&self) -> test_delay::CodecFormat; } pub struct DecoderCfg { diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 70d944a33..945de6335 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -6,7 +6,7 @@ use hbb_common::{ anyhow::{anyhow, Context}, config::HwCodecConfig, lazy_static, log, - message_proto::{test_delay, H264s, H265s, Message, VideoFrame, H264, H265}, + message_proto::{H264s, H265s, Message, VideoFrame, H264, H265}, ResultType, }; use hwcodec::{ @@ -143,13 +143,6 @@ impl EncoderApi for HwEncoder { self.encoder.set_bitrate((bitrate * 1000) as _).ok(); Ok(()) } - - fn get_codec_format(&self) -> test_delay::CodecFormat { - match self.format { - DataFormat::H264 => test_delay::CodecFormat::H264, - DataFormat::H265 => test_delay::CodecFormat::H265, - } - } } impl HwEncoder { diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index b9b95b93e..2943419e4 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -3,7 +3,7 @@ // https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs use hbb_common::anyhow::{anyhow, Context}; -use hbb_common::message_proto::{test_delay, Message, VP9s, VideoFrame, VP9}; +use hbb_common::message_proto::{Message, VP9s, VideoFrame, VP9}; use hbb_common::ResultType; use crate::codec::EncoderApi; @@ -27,7 +27,6 @@ impl Default for VpxVideoCodecId { pub struct VpxEncoder { ctx: vpx_codec_ctx_t, - format: VpxVideoCodecId, width: usize, height: usize, } @@ -97,17 +96,14 @@ impl EncoderApi for VpxEncoder { { match cfg { crate::codec::EncoderCfg::VPX(config) => { - let format; let i; if cfg!(feature = "VP8") { i = match config.codec { VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()), VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()), }; - format = config.codec; } else { i = call_vpx_ptr!(vpx_codec_vp9_cx()); - format = VpxVideoCodecId::VP9; } let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0)); @@ -194,7 +190,6 @@ impl EncoderApi for VpxEncoder { Ok(Self { ctx, - format, width: config.width as _, height: config.height as _, }) @@ -233,13 +228,6 @@ impl EncoderApi for VpxEncoder { call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg)); return Ok(()); } - - fn get_codec_format(&self) -> test_delay::CodecFormat { - match self.format { - VpxVideoCodecId::VP8 => test_delay::CodecFormat::VP8, - VpxVideoCodecId::VP9 => test_delay::CodecFormat::VP9, - } - } } impl VpxEncoder { diff --git a/src/client.rs b/src/client.rs index 48495d184..d63ce970c 100644 --- a/src/client.rs +++ b/src/client.rs @@ -40,7 +40,7 @@ pub use super::lang::*; pub mod file_trait; pub use file_trait::FileManager; pub mod helper; -pub use helper::LatencyController; +pub use helper::*; pub const SEC30: Duration = Duration::from_secs(30); pub struct Client; diff --git a/src/client/helper.rs b/src/client/helper.rs index abd20d312..ea12cb7ee 100644 --- a/src/client/helper.rs +++ b/src/client/helper.rs @@ -3,7 +3,7 @@ use std::{ time::Instant, }; -use hbb_common::log; +use hbb_common::{log, message_proto::{VideoFrame, video_frame}}; const MAX_LATENCY: i64 = 500; const MIN_LATENCY: i64 = 100; @@ -57,3 +57,33 @@ impl LatencyController { self.allow_audio } } + +#[derive(PartialEq, Debug, Clone)] +pub enum CodecFormat { + VP9, + H264, + H265, + Unknown, +} + +impl From<&VideoFrame> for CodecFormat { + fn from(it: &VideoFrame) -> Self { + match it.union { + Some(video_frame::Union::vp9s(_)) => CodecFormat::VP9, + Some(video_frame::Union::h264s(_)) => CodecFormat::H264, + Some(video_frame::Union::h265s(_)) => CodecFormat::H265, + _ => CodecFormat::Unknown, + } + } +} + +impl ToString for CodecFormat { + fn to_string(&self) -> String { + match self { + CodecFormat::VP9 => "VP9".into(), + CodecFormat::H264 => "H264".into(), + CodecFormat::H265 => "H265".into(), + CodecFormat::Unknown => "Unknow".into(), + } + } +} diff --git a/src/server/connection.rs b/src/server/connection.rs index f8bf5d16a..46c730092 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -380,7 +380,6 @@ impl Connection { time, last_delay:qos.current_delay, target_bitrate:qos.target_bitrate, - codec_format:qos.codec_format.into(), ..Default::default() }); conn.inner.send(msg_out.into()); diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 355a43305..3b770d250 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -73,7 +73,6 @@ pub struct VideoQoS { user_image_quality: u32, current_image_quality: u32, enable_abr: bool, - pub codec_format: test_delay::CodecFormat, pub current_delay: u32, pub fps: u8, // abr pub target_bitrate: u32, // abr @@ -111,7 +110,6 @@ impl Default for VideoQoS { user_image_quality: ImageQuality::Balanced.as_percent(), current_image_quality: ImageQuality::Balanced.as_percent(), enable_abr: false, - codec_format: Default::default(), width: 0, height: 0, current_delay: 0, @@ -564,7 +562,7 @@ fn run(sp: GenericService) -> ResultType<()> { let mut spf = video_qos.spf(); let bitrate = video_qos.generate_bitrate()?; let abr = video_qos.check_abr_config(); - + drop(video_qos); log::info!("init bitrate={}, abr enabled:{}", bitrate, abr); let encoder_cfg = match Encoder::current_hw_encoder_name() { @@ -589,10 +587,7 @@ fn run(sp: GenericService) -> ResultType<()> { Ok(x) => encoder = x, Err(err) => bail!("Failed to create encoder: {}", err), } - - video_qos.codec_format = encoder.get_codec_format(); - drop(video_qos); - + let privacy_mode_id = *PRIVACY_MODE_CONN_ID.lock().unwrap(); #[cfg(not(windows))] let captuerer_privacy_mode_id = privacy_mode_id; diff --git a/src/ui/remote.rs b/src/ui/remote.rs index 1dd685f15..90f63b554 100644 --- a/src/ui/remote.rs +++ b/src/ui/remote.rs @@ -234,25 +234,13 @@ impl sciter::EventHandler for Handler { } } -#[derive(Debug)] +#[derive(Debug, Default)] struct QualityStatus { - speed: String, - fps: i32, - delay: i32, - target_bitrate: i32, - codec_format: test_delay::CodecFormat, -} - -impl Default for QualityStatus { - fn default() -> Self { - Self { - speed: Default::default(), - fps: -1, - delay: -1, - target_bitrate: -1, - codec_format: test_delay::CodecFormat::Unknown, - } - } + speed: Option, + fps: Option, + delay: Option, + target_bitrate: Option, + codec_format: Option, } impl Handler { @@ -271,21 +259,16 @@ impl Handler { } fn update_quality_status(&self, status: QualityStatus) { - let codec_format = match status.codec_format { - test_delay::CodecFormat::Unknown => "Unknown", - test_delay::CodecFormat::VP8 => "VP8", - test_delay::CodecFormat::VP9 => "VP9", - test_delay::CodecFormat::H264 => "H264", - test_delay::CodecFormat::H265 => "H265", - }; self.call2( "updateQualityStatus", &make_args!( - status.speed, - status.fps, - status.delay, - status.target_bitrate, - codec_format + status.speed.map_or(Value::null(), |it| it.into()), + status.fps.map_or(Value::null(), |it| it.into()), + status.delay.map_or(Value::null(), |it| it.into()), + status.target_bitrate.map_or(Value::null(), |it| it.into()), + status + .codec_format + .map_or(Value::null(), |it| it.to_string().into()) ), ); } @@ -1365,6 +1348,7 @@ async fn io_loop(handler: Handler) { clipboard_file_context: None, data_count: Arc::new(AtomicUsize::new(0)), frame_count, + video_format: CodecFormat::Unknown, }; remote.io_loop(&key, &token).await; remote.sync_jobs_status_to_local().await; @@ -1417,6 +1401,7 @@ struct Remote { clipboard_file_context: Option>, data_count: Arc, frame_count: Arc, + video_format: CodecFormat, } impl Remote { @@ -1506,8 +1491,8 @@ impl Remote { let speed = format!("{:.2}kB/s", speed as f32 / 1024 as f32); let fps = self.frame_count.swap(0, Ordering::Relaxed) as _; self.handler.update_quality_status(QualityStatus { - speed, - fps, + speed:Some(speed), + fps:Some(fps), ..Default::default() }); } @@ -2038,6 +2023,14 @@ impl Remote { self.handler.call2("closeSuccess", &make_args!()); self.handler.call("adaptSize", &make_args!()); } + let incomming_format = CodecFormat::from(&vf); + if self.video_format != incomming_format { + self.video_format = incomming_format.clone(); + self.handler.update_quality_status(QualityStatus { + codec_format: Some(incomming_format), + ..Default::default() + }) + }; self.video_sender.send(MediaData::VideoFrame(vf)).ok(); } Some(message::Union::hash(hash)) => { @@ -2612,9 +2605,8 @@ impl Interface for Handler { async fn handle_test_delay(&mut self, t: TestDelay, peer: &mut Stream) { if !t.from_client { self.update_quality_status(QualityStatus { - delay: t.last_delay as _, - target_bitrate: t.target_bitrate as _, - codec_format: t.codec_format.enum_value_or_default(), + delay: Some(t.last_delay as _), + target_bitrate: Some(t.target_bitrate as _), ..Default::default() }); handle_test_delay(t, peer).await; diff --git a/src/ui/remote.tis b/src/ui/remote.tis index dc2470ac1..8bc29ac60 100644 --- a/src/ui/remote.tis +++ b/src/ui/remote.tis @@ -492,10 +492,10 @@ class QualityMonitor: Reactor.Component $(#quality-monitor).content(); handler.updateQualityStatus = function(speed, fps, delay, bitrate, codec_format) { speed ? qualityMonitorData[0] = speed:null; - fps > -1 ? qualityMonitorData[1] = fps:null; - delay > -1 ? qualityMonitorData[2] = delay:null; - bitrate > -1 ? qualityMonitorData[3] = bitrate:null; - codec_format != "Unknown" ? qualityMonitorData[4] = codec_format:null; + fps ? qualityMonitorData[1] = fps:null; + delay ? qualityMonitorData[2] = delay:null; + bitrate ? qualityMonitorData[3] = bitrate:null; + codec_format ? qualityMonitorData[4] = codec_format:null; qualityMonitor.update(); }