From 2133f910899629d88ad59d89a4d3de119129c82e Mon Sep 17 00:00:00 2001 From: 21pages Date: Wed, 19 Jul 2023 13:11:24 +0800 Subject: [PATCH] codec set quality seperately and refactor network delay Signed-off-by: 21pages --- libs/scrap/examples/benchmark.rs | 46 +++-- libs/scrap/examples/record-screen.rs | 36 ++-- libs/scrap/src/common/aom.rs | 79 ++++++-- libs/scrap/src/common/codec.rs | 36 +++- libs/scrap/src/common/hwcodec.rs | 39 +++- libs/scrap/src/common/vpxcodec.rs | 78 +++++++- src/server/connection.rs | 15 +- src/server/video_qos.rs | 277 +++++++++++++++++---------- src/server/video_service.rs | 32 ++-- 9 files changed, 457 insertions(+), 181 deletions(-) diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index eccf502d9..c121636d3 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -2,7 +2,7 @@ use docopt::Docopt; use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV}; use scrap::{ aom::{AomDecoder, AomDecoderConfig, AomEncoder, AomEncoderConfig}, - codec::{EncoderApi, EncoderCfg}, + codec::{EncoderApi, EncoderCfg, Quality as Q}, Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, VpxVideoCodecId::{self, *}, STRIDE_ALIGN, @@ -15,13 +15,14 @@ const USAGE: &'static str = " Codec benchmark. Usage: - benchmark [--count=COUNT] [--bitrate=KBS] [--hw-pixfmt=PIXFMT] + benchmark [--count=COUNT] [--quality=QUALITY] [--hw-pixfmt=PIXFMT] benchmark (-h | --help) Options: -h --help Show this screen. --count=COUNT Capture frame count [default: 100]. - --bitrate=KBS Video bitrate in kilobits per second [default: 5000]. + --quality=QUALITY Video quality [default: Balanced]. + Valid values: Best, Balanced, Low. --hw-pixfmt=PIXFMT Hardware codec pixfmt. [default: i420] Valid values: i420, nv12. "; @@ -29,7 +30,7 @@ Options: #[derive(Debug, serde::Deserialize)] struct Args { flag_count: usize, - flag_bitrate: usize, + flag_quality: Quality, flag_hw_pixfmt: Pixfmt, } @@ -39,20 +40,32 @@ enum Pixfmt { NV12, } +#[derive(Debug, serde::Deserialize)] +enum Quality { + Best, + Balanced, + Low, +} + fn main() { init_from_env(Env::default().filter_or(DEFAULT_FILTER_ENV, "info")); let args: Args = Docopt::new(USAGE) .and_then(|d| d.deserialize()) .unwrap_or_else(|e| e.exit()); - let bitrate_k = args.flag_bitrate; + let quality = args.flag_quality; let yuv_count = args.flag_count; let (yuvs, width, height) = capture_yuv(yuv_count); println!( - "benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}", - width, height, bitrate_k, args.flag_hw_pixfmt + "benchmark {}x{} quality:{:?}k hw_pixfmt:{:?}", + width, height, quality, args.flag_hw_pixfmt ); - [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count)); - test_av1(&yuvs, width, height, bitrate_k, yuv_count); + let quality = match quality { + Quality::Best => Q::Best, + Quality::Balanced => Q::Balanced, + Quality::Low => Q::Low, + }; + [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, quality, yuv_count)); + test_av1(&yuvs, width, height, quality, yuv_count); #[cfg(feature = "hwcodec")] { use hwcodec::AVPixelFormat; @@ -61,7 +74,7 @@ fn main() { Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12, }; let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt); - hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt); + hw::test(&yuvs, width, height, quality, yuv_count, hw_pixfmt); } } @@ -95,14 +108,14 @@ fn test_vpx( yuvs: &Vec>, width: usize, height: usize, - bitrate_k: usize, + quality: Q, yuv_count: usize, ) { let config = EncoderCfg::VPX(VpxEncoderConfig { width: width as _, height: height as _, timebase: [1, 1000], - bitrate: bitrate_k as _, + quality, codec: codec_id, num_threads: (num_cpus::get() / 2) as _, }); @@ -148,11 +161,11 @@ fn test_vpx( ); } -fn test_av1(yuvs: &Vec>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) { +fn test_av1(yuvs: &Vec>, width: usize, height: usize, quality: Q, yuv_count: usize) { let config = EncoderCfg::AOM(AomEncoderConfig { width: width as _, height: height as _, - bitrate: bitrate_k as _, + quality, }); let mut encoder = AomEncoder::new(config).unwrap(); let start = Instant::now(); @@ -208,17 +221,18 @@ mod hw { yuvs: &Vec>, width: usize, height: usize, - bitrate_k: usize, + quality: Q, yuv_count: usize, pixfmt: AVPixelFormat, ) { + let bitrate = scrap::hwcodec::HwEncoder::convert_quality(quality); let ctx = EncodeContext { name: String::from(""), width: width as _, height: height as _, pixfmt, align: 0, - bitrate: (bitrate_k * 1000) as _, + bitrate: bitrate as i32 * 1000, timebase: [1, 30], gop: 60, quality: Quality_Default, diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 10ad66e09..5b8d40d4a 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -13,7 +13,7 @@ use std::time::{Duration, Instant}; use std::{io, thread}; use docopt::Docopt; -use scrap::codec::{EncoderApi, EncoderCfg}; +use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q}; use webm::mux; use webm::mux::Track; @@ -24,17 +24,18 @@ const USAGE: &'static str = " Simple WebM screen capture. Usage: - record-screen [--time=] [--fps=] [--bv=] [--ba=] [--codec CODEC] + record-screen [--time=] [--fps=] [--quality=] [--ba=] [--codec CODEC] record-screen (-h | --help) Options: - -h --help Show this screen. - --time= Recording duration in seconds. - --fps= Frames per second [default: 30]. - --bv= Video bitrate in kilobits per second [default: 5000]. - --ba= Audio bitrate in kilobits per second [default: 96]. - --codec CODEC Configure the codec used. [default: vp9] - Valid values: vp8, vp9. + -h --help Show this screen. + --time= Recording duration in seconds. + --fps= Frames per second [default: 30]. + --quality= Video quality [default: Balanced]. + Valid values: Best, Balanced, Low. + --ba= Audio bitrate in kilobits per second [default: 96]. + --codec CODEC Configure the codec used. [default: vp9] + Valid values: vp8, vp9. "; #[derive(Debug, serde::Deserialize)] @@ -43,7 +44,14 @@ struct Args { flag_codec: Codec, flag_time: Option, flag_fps: u64, - flag_bv: u32, + flag_quality: Quality, +} + +#[derive(Debug, serde::Deserialize)] +enum Quality { + Best, + Balanced, + Low, } #[derive(Debug, serde::Deserialize)] @@ -97,12 +105,16 @@ fn main() -> io::Result<()> { let mut vt = webm.add_video_track(width, height, None, mux_codec); // Setup the encoder. - + let quality = match args.flag_quality { + Quality::Best => Q::Best, + Quality::Balanced => Q::Balanced, + Quality::Low => Q::Low, + }; let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig { width, height, timebase: [1, 1000], - bitrate: args.flag_bv, + quality, codec: vpx_codec, num_threads: 0, })) diff --git a/libs/scrap/src/common/aom.rs b/libs/scrap/src/common/aom.rs index b9748efda..a2064e4c2 100644 --- a/libs/scrap/src/common/aom.rs +++ b/libs/scrap/src/common/aom.rs @@ -6,6 +6,7 @@ include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs")); +use crate::codec::{base_bitrate, Quality}; use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN}; use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result}; use hbb_common::{ @@ -43,7 +44,7 @@ impl Default for aom_image_t { pub struct AomEncoderConfig { pub width: u32, pub height: u32, - pub bitrate: u32, + pub quality: Quality, } pub struct AomEncoder { @@ -56,7 +57,6 @@ pub struct AomEncoder { mod webrtc { use super::*; - const kQpMin: u32 = 10; const kUsageProfile: u32 = AOM_USAGE_REALTIME; const kMinQindex: u32 = 145; // Min qindex threshold for QP scaling. const kMaxQindex: u32 = 205; // Max qindex threshold for QP scaling. @@ -65,7 +65,8 @@ mod webrtc { const kRtpTicksPerSecond: i32 = 90000; const kMinimumFrameRate: f64 = 1.0; - const kQpMax: u32 = 25; // to-do: webrtc use dynamic value, no more than 63 + pub const DEFAULT_Q_MAX: u32 = 56; // no more than 63 + pub const DEFAULT_Q_MIN: u32 = 12; // no more than 63, litter than q_max fn number_of_threads(width: u32, height: u32, number_of_cores: usize) -> u32 { // Keep the number of encoder threads equal to the possible number of @@ -78,7 +79,7 @@ mod webrtc { } else { // Use 2 threads for low res on ARM. #[cfg(any(target_arch = "arm", target_arch = "aarch64", target_os = "android"))] - if (width * height >= 320 * 180 && number_of_cores > 2) { + if width * height >= 320 * 180 && number_of_cores > 2 { return 2; } // 1 thread less than VGA. @@ -122,11 +123,23 @@ mod webrtc { c.g_threads = number_of_threads(cfg.width, cfg.height, num_cpus::get()); c.g_timebase.num = 1; c.g_timebase.den = kRtpTicksPerSecond; - c.rc_target_bitrate = cfg.bitrate; // kilobits/sec. c.g_input_bit_depth = kBitDepth; c.kf_mode = aom_kf_mode::AOM_KF_DISABLED; - c.rc_min_quantizer = kQpMin; - c.rc_max_quantizer = kQpMax; + let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality); + if q_min > 0 && q_min < q_max && q_max < 64 { + c.rc_min_quantizer = q_min; + c.rc_max_quantizer = q_max; + } else { + c.rc_min_quantizer = DEFAULT_Q_MIN; + c.rc_max_quantizer = DEFAULT_Q_MAX; + } + let base_bitrate = base_bitrate(cfg.width as _, cfg.height as _); + let bitrate = base_bitrate * b / 100; + if bitrate > 0 { + c.rc_target_bitrate = bitrate; + } else { + c.rc_target_bitrate = base_bitrate; + } c.rc_undershoot_pct = 50; c.rc_overshoot_pct = 50; c.rc_buf_initial_sz = 600; @@ -259,11 +272,24 @@ impl EncoderApi for AomEncoder { true } - fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> { - let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() }; - new_enc_cfg.rc_target_bitrate = bitrate; - call_aom!(aom_codec_enc_config_set(&mut self.ctx, &new_enc_cfg)); - return Ok(()); + fn set_quality(&mut self, quality: Quality) -> ResultType<()> { + let mut c = unsafe { *self.ctx.config.enc.to_owned() }; + let (q_min, q_max, b) = Self::convert_quality(quality); + if q_min > 0 && q_min < q_max && q_max < 64 { + c.rc_min_quantizer = q_min; + c.rc_max_quantizer = q_max; + } + let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100; + if bitrate > 0 { + c.rc_target_bitrate = bitrate; + } + call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c)); + Ok(()) + } + + fn bitrate(&self) -> u32 { + let c = unsafe { *self.ctx.config.enc.to_owned() }; + c.rc_target_bitrate } } @@ -319,6 +345,35 @@ impl AomEncoder { ..Default::default() } } + + pub fn convert_quality(quality: Quality) -> (u32, u32, u32) { + // we can use lower bitrate for av1 + match quality { + Quality::Best => (12, 25, 100), + Quality::Balanced => (12, 35, 100 * 2 / 3), + Quality::Low => (18, 45, 50), + Quality::Custom(b) => { + let (q_min, q_max) = Self::calc_q_values(b); + (q_min, q_max, b) + } + } + } + + #[inline] + fn calc_q_values(b: u32) -> (u32, u32) { + let b = std::cmp::min(b, 200); + let q_min1: i32 = 24; + let q_min2 = 12; + let q_max1 = 45; + let q_max2 = 25; + + let t = b as f32 / 200.0; + + let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32; + let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32; + + (q_min, q_max) + } } impl Drop for AomEncoder { diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 03e8a19a2..90b65c6cf 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -42,7 +42,7 @@ pub struct HwEncoderConfig { pub name: String, pub width: usize, pub height: usize, - pub bitrate: i32, + pub quality: Quality, } #[derive(Debug, Clone)] @@ -61,7 +61,9 @@ pub trait EncoderApi { fn use_yuv(&self) -> bool; - fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>; + fn set_quality(&mut self, quality: Quality) -> ResultType<()>; + + fn bitrate(&self) -> u32; } pub struct Encoder { @@ -471,3 +473,33 @@ fn enable_hwcodec_option() -> bool { } return true; // default is true } + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Quality { + Best, + Balanced, + Low, + Custom(u32), +} + +impl Default for Quality { + fn default() -> Self { + Self::Balanced + } +} + +pub fn base_bitrate(width: u32, height: u32) -> u32 { + #[allow(unused_mut)] + let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9 + if base_bitrate == 0 { + base_bitrate = 1920 * 1080 / 1000; + } + #[cfg(target_os = "android")] + { + // fix when android screen shrinks + let fix = crate::Display::fix_quality() as u32; + log::debug!("Android screen, fix quality:{}", fix); + base_bitrate = base_bitrate * fix; + } + base_bitrate +} diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 50ad33099..76c475970 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -1,5 +1,5 @@ use crate::{ - codec::{EncoderApi, EncoderCfg}, + codec::{base_bitrate, EncoderApi, EncoderCfg}, hw, ImageFormat, ImageRgb, HW_STRIDE_ALIGN, }; use hbb_common::{ @@ -34,6 +34,9 @@ pub struct HwEncoder { yuv: Vec, pub format: DataFormat, pub pixfmt: AVPixelFormat, + width: u32, + height: u32, + bitrate: u32, //kbs } impl EncoderApi for HwEncoder { @@ -43,13 +46,19 @@ impl EncoderApi for HwEncoder { { match cfg { EncoderCfg::HW(config) => { + let b = Self::convert_quality(config.quality); + let base_bitrate = base_bitrate(config.width as _, config.height as _); + let mut bitrate = base_bitrate * b / 100; + if base_bitrate <= 0 { + bitrate = base_bitrate; + } let ctx = EncodeContext { name: config.name.clone(), width: config.width as _, height: config.height as _, pixfmt: DEFAULT_PIXFMT, align: HW_STRIDE_ALIGN as _, - bitrate: config.bitrate * 1000, + bitrate: bitrate as i32 * 1000, timebase: DEFAULT_TIME_BASE, gop: DEFAULT_GOP, quality: DEFAULT_HW_QUALITY, @@ -70,6 +79,9 @@ impl EncoderApi for HwEncoder { yuv: vec![], format, pixfmt: ctx.pixfmt, + width: ctx.width as _, + height: ctx.height as _, + bitrate, }), Err(_) => Err(anyhow!(format!("Failed to create encoder"))), } @@ -114,10 +126,19 @@ impl EncoderApi for HwEncoder { false } - fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> { - self.encoder.set_bitrate((bitrate * 1000) as _).ok(); + fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> { + let b = Self::convert_quality(quality); + let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100; + if bitrate > 0 { + self.encoder.set_bitrate((bitrate * 1000) as _).ok(); + self.bitrate = bitrate; + } Ok(()) } + + fn bitrate(&self) -> u32 { + self.bitrate + } } impl HwEncoder { @@ -159,6 +180,16 @@ impl HwEncoder { Err(_) => Ok(Vec::::new()), } } + + pub fn convert_quality(quality: crate::codec::Quality) -> u32 { + use crate::codec::Quality; + match quality { + Quality::Best => 150, + Quality::Balanced => 100, + Quality::Low => 50, + Quality::Custom(b) => b, + } + } } pub struct HwDecoder { diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index 801527811..1717047b8 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -7,7 +7,7 @@ use hbb_common::log; use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}; use hbb_common::ResultType; -use crate::codec::EncoderApi; +use crate::codec::{base_bitrate, EncoderApi, Quality}; use crate::{GoogleImage, STRIDE_ALIGN}; use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; @@ -19,6 +19,9 @@ use std::{ptr, slice}; generate_call_macro!(call_vpx, false); generate_call_ptr_macro!(call_vpx_ptr); +const DEFAULT_QP_MAX: u32 = 56; // no more than 63 +const DEFAULT_QP_MIN: u32 = 12; // no more than 63 + #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)] pub enum VpxVideoCodecId { VP8, @@ -64,8 +67,9 @@ impl EncoderApi for VpxEncoder { c.g_h = config.height; c.g_timebase.num = config.timebase[0]; c.g_timebase.den = config.timebase[1]; - c.rc_target_bitrate = config.bitrate; c.rc_undershoot_pct = 95; + // When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature. + // In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps. c.rc_dropframe_thresh = 25; c.g_threads = if config.num_threads == 0 { num_cpus::get() as _ @@ -79,6 +83,21 @@ impl EncoderApi for VpxEncoder { // c.kf_min_dist = 0; // c.kf_max_dist = 999999; c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot + let (q_min, q_max, b) = Self::convert_quality(config.quality); + if q_min > 0 && q_min < q_max && q_max < 64 { + c.rc_min_quantizer = q_min; + c.rc_max_quantizer = q_max; + } else { + c.rc_min_quantizer = DEFAULT_QP_MIN; + c.rc_max_quantizer = DEFAULT_QP_MAX; + } + let base_bitrate = base_bitrate(config.width as _, config.height as _); + let bitrate = base_bitrate * b / 100; + if bitrate > 0 { + c.rc_target_bitrate = bitrate; + } else { + c.rc_target_bitrate = base_bitrate; + } /* The VPX encoder supports two-pass encoding for rate control purposes. @@ -177,11 +196,24 @@ impl EncoderApi for VpxEncoder { true } - fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> { - let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() }; - new_enc_cfg.rc_target_bitrate = bitrate; - call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg)); - return Ok(()); + fn set_quality(&mut self, quality: Quality) -> ResultType<()> { + let mut c = unsafe { *self.ctx.config.enc.to_owned() }; + let (q_min, q_max, b) = Self::convert_quality(quality); + if q_min > 0 && q_min < q_max && q_max < 64 { + c.rc_min_quantizer = q_min; + c.rc_max_quantizer = q_max; + } + let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100; + if bitrate > 0 { + c.rc_target_bitrate = bitrate; + } + call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c)); + Ok(()) + } + + fn bitrate(&self) -> u32 { + let c = unsafe { *self.ctx.config.enc.to_owned() }; + c.rc_target_bitrate } } @@ -258,6 +290,34 @@ impl VpxEncoder { ..Default::default() } } + + fn convert_quality(quality: Quality) -> (u32, u32, u32) { + match quality { + Quality::Best => (6, 45, 150), + Quality::Balanced => (12, 56, 100 * 2 / 3), + Quality::Low => (18, 56, 50), + Quality::Custom(b) => { + let (q_min, q_max) = Self::calc_q_values(b); + (q_min, q_max, b) + } + } + } + + #[inline] + fn calc_q_values(b: u32) -> (u32, u32) { + let b = std::cmp::min(b, 200); + let q_min1: i32 = 36; + let q_min2 = 12; + let q_max1 = 56; + let q_max2 = 37; + + let t = b as f32 / 200.0; + + let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32; + let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32; + + (q_min, q_max) + } } impl Drop for VpxEncoder { @@ -289,8 +349,8 @@ pub struct VpxEncoderConfig { pub height: c_uint, /// The timebase numerator and denominator (in seconds). pub timebase: [c_int; 2], - /// The target bitrate (in kilobits per second). - pub bitrate: c_uint, + /// The image quality + pub quality: Quality, /// The codec pub codec: VpxVideoCodecId, pub num_threads: u32, diff --git a/src/server/connection.rs b/src/server/connection.rs index 4f2321f9d..84f540027 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -196,6 +196,7 @@ pub struct Connection { #[cfg(not(any(feature = "flatpak", feature = "appimage")))] tx_desktop_ready: mpsc::Sender<()>, closed: bool, + delay_response_instant: Instant, } impl ConnInner { @@ -326,6 +327,7 @@ impl Connection { #[cfg(not(any(feature = "flatpak", feature = "appimage")))] tx_desktop_ready: _tx_desktop_ready, closed: false, + delay_response_instant: Instant::now(), }; if !conn.on_open(addr).await { conn.closed = true; @@ -593,18 +595,19 @@ impl Connection { break; } let time = get_time(); + let mut qos = video_service::VIDEO_QOS.lock().unwrap(); if time > 0 && conn.last_test_delay == 0 { conn.last_test_delay = time; let mut msg_out = Message::new(); - let qos = video_service::VIDEO_QOS.lock().unwrap(); msg_out.set_test_delay(TestDelay{ time, last_delay:conn.network_delay.unwrap_or_default(), - target_bitrate:qos.bitrate(), + target_bitrate: qos.bitrate(), ..Default::default() }); conn.inner.send(msg_out.into()); } + qos.user_delay_response_elapsed(conn.inner.id(), conn.delay_response_instant.elapsed().as_millis()); } } } @@ -1188,7 +1191,9 @@ impl Connection { #[inline] #[cfg(not(any(target_os = "android", target_os = "ios")))] fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) { - self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok(); + self.tx_input + .send(MessageInput::Pointer((msg, conn_id))) + .ok(); } #[inline] @@ -1553,6 +1558,7 @@ impl Connection { .unwrap() .user_network_delay(self.inner.id(), new_delay); self.network_delay = Some(new_delay); + self.delay_response_instant = Instant::now(); } } else if let Some(message::Union::SwitchSidesResponse(_s)) = msg.union { #[cfg(feature = "flutter")] @@ -1590,7 +1596,8 @@ impl Connection { self.input_mouse(me, self.inner.id()); } } - Some(message::Union::PointerDeviceEvent(pde)) => { + Some(message::Union::PointerDeviceEvent(pde)) => + { #[cfg(not(any(target_os = "android", target_os = "ios")))] if self.peer_keyboard_enabled() { MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst); diff --git a/src/server/video_qos.rs b/src/server/video_qos.rs index 727c1ea35..18b5f50de 100644 --- a/src/server/video_qos.rs +++ b/src/server/video_qos.rs @@ -1,4 +1,5 @@ use super::*; +use scrap::codec::Quality; use std::time::Duration; pub const FPS: u32 = 30; pub const MIN_FPS: u32 = 1; @@ -18,21 +19,29 @@ impl Percent for ImageQuality { } } -#[derive(Default, Debug)] +#[derive(Default, Debug, Copy, Clone)] +struct Delay { + state: DelayState, + staging_state: DelayState, + delay: u32, + counter: u32, + slower_than_old_state: Option, +} + +#[derive(Default, Debug, Copy, Clone)] struct UserData { full_speed_fps: Option, custom_fps: Option, - quality: Option<(i32, i64)>, // (quality, time) - delay: Option<(DelayState, u32, usize)>, // (state, ms, counter) + quality: Option<(i64, Quality)>, // (time, quality) + delay: Option, + response_delayed: bool, } pub struct VideoQoS { - width: u32, - height: u32, fps: u32, - target_bitrate: u32, - updated: bool, + quality: Quality, users: HashMap, + bitrate_store: u32, } #[derive(PartialEq, Debug, Clone, Copy)] @@ -43,6 +52,12 @@ enum DelayState { Broken = 1000, } +impl Default for DelayState { + fn default() -> Self { + DelayState::Normal + } +} + impl DelayState { fn from_delay(delay: u32) -> Self { if delay > DelayState::Broken as u32 { @@ -61,24 +76,19 @@ impl Default for VideoQoS { fn default() -> Self { VideoQoS { fps: FPS, - width: 0, - height: 0, - target_bitrate: 0, - updated: false, + quality: Default::default(), users: Default::default(), + bitrate_store: 0, } } } -impl VideoQoS { - pub fn set_size(&mut self, width: u32, height: u32) { - if width == 0 || height == 0 { - return; - } - self.width = width; - self.height = height; - } +#[derive(Debug, PartialEq, Eq)] +pub enum RefreshType { + SetImageQuality, +} +impl VideoQoS { pub fn spf(&self) -> Duration { Duration::from_secs_f32(1. / (self.fps as f32)) } @@ -87,24 +97,23 @@ impl VideoQoS { self.fps } - pub fn bitrate(&self) -> u32 { - self.target_bitrate + pub fn store_bitrate(&mut self, bitrate: u32) { + self.bitrate_store = bitrate; } - pub fn check_if_updated(&mut self) -> bool { - if self.updated { - self.updated = false; - return true; - } - return false; + pub fn bitrate(&self) -> u32 { + self.bitrate_store + } + + pub fn quality(&self) -> Quality { + self.quality } pub fn abr_enabled() -> bool { "N" != Config::get_option("enable-abr") } - pub fn refresh(&mut self) { - let mut updated = false; + pub fn refresh(&mut self, typ: Option) { // fps let user_fps = |u: &UserData| { // full_speed_fps @@ -117,13 +126,19 @@ impl VideoQoS { } // delay if let Some(delay) = u.delay { - fps = match delay.0 { + fps = match delay.state { DelayState::Normal => fps, DelayState::LowDelay => fps, DelayState::HighDelay => fps / 2, DelayState::Broken => fps / 4, } } + // delay response + if u.response_delayed { + if fps > MIN_FPS + 2 { + fps = MIN_FPS + 2; + } + } return fps; }; let mut fps = self @@ -136,64 +151,79 @@ impl VideoQoS { if fps > MAX_FPS { fps = MAX_FPS; } - if fps != self.fps { - self.fps = fps; - updated = true; - } + self.fps = fps; // quality // latest image quality - let latest = self + let latest_quality = self .users .iter() - // .map(|(_, u)| u.quality) - .filter(|u| u.1.quality != None) - .max_by(|u1, u2| { - u1.1.quality - .unwrap_or_default() - .1 - .cmp(&u2.1.quality.unwrap_or_default().1) - }); - let quality = if let Some((id, data)) = latest { - let mut quality = data.quality.unwrap_or_default().0; - if quality <= 0 { - quality = ImageQuality::Balanced.as_percent() as _; - } - // use latest's delay for quality - if Self::abr_enabled() { - if let Some(Some((delay, _, _))) = self.users.get(id).map(|u| u.delay) { - quality = match delay { - DelayState::Normal => quality, - DelayState::LowDelay => std::cmp::min(quality, 50), - DelayState::HighDelay => std::cmp::min(quality, 25), - DelayState::Broken => 10, - }; + .map(|(_, u)| u.quality) + .filter(|q| *q != None) + .max_by(|a, b| a.unwrap_or_default().0.cmp(&b.unwrap_or_default().0)) + .unwrap_or_default() + .unwrap_or_default() + .1; + let mut quality = latest_quality; + + // network delay + if Self::abr_enabled() && typ != Some(RefreshType::SetImageQuality) { + // max delay + let delay = self + .users + .iter() + .map(|u| u.1.delay) + .filter(|d| d.is_some()) + .max_by(|a, b| { + (a.unwrap_or_default().state as u32).cmp(&(b.unwrap_or_default().state as u32)) + }); + let delay = delay.unwrap_or_default().unwrap_or_default().state; + if delay != DelayState::Normal { + match self.quality { + Quality::Best => { + quality = Quality::Balanced; + } + Quality::Balanced => { + quality = Quality::Low; + } + Quality::Low => { + quality = Quality::Low; + } + Quality::Custom(b) => match delay { + DelayState::LowDelay => { + quality = + Quality::Custom(if b >= 150 { 100 } else { std::cmp::min(50, b) }); + } + DelayState::HighDelay => { + quality = + Quality::Custom(if b >= 100 { 50 } else { std::cmp::min(25, b) }); + } + DelayState::Broken => { + quality = + Quality::Custom(if b >= 50 { 25 } else { std::cmp::min(10, b) }); + } + DelayState::Normal => {} + }, + } + } else { + match self.quality { + Quality::Low => { + if latest_quality == Quality::Best { + quality = Quality::Balanced; + } + } + Quality::Custom(current_b) => { + if let Quality::Custom(latest_b) = latest_quality { + if current_b < latest_b / 2 { + quality = Quality::Custom(latest_b / 2); + } + } + } + _ => {} } } - quality - } else { - ImageQuality::Balanced.as_percent() as _ - }; - // bitrate - #[allow(unused_mut)] - let mut base_bitrate = ((self.width * self.height) / 800) as u32; - if base_bitrate == 0 { - base_bitrate = 1920 * 1080 / 800; } - #[cfg(target_os = "android")] - { - // fix when android screen shrinks - let fix = scrap::Display::fix_quality() as u32; - log::debug!("Android screen, fix quality:{}", fix); - base_bitrate = base_bitrate * fix; - } - let target_bitrate = base_bitrate * quality as u32 / 100; - if self.target_bitrate != target_bitrate { - self.target_bitrate = target_bitrate; - updated = true; - } - - self.updated = updated; + self.quality = quality; } pub fn user_custom_fps(&mut self, id: i32, fps: u32) { @@ -211,7 +241,7 @@ impl VideoQoS { }, ); } - self.refresh(); + self.refresh(None); } pub fn user_full_speed_fps(&mut self, id: i32, full_speed_fps: u32) { @@ -226,23 +256,27 @@ impl VideoQoS { }, ); } - self.refresh(); + self.refresh(None); } pub fn user_image_quality(&mut self, id: i32, image_quality: i32) { - let convert_quality = |q: i32| -> i32 { + // https://github.com/rustdesk/rustdesk/blob/d716e2b40c38737f1aa3f16de0dec67394a6ac68/src/server/video_service.rs#L493 + let convert_quality = |q: i32| { if q == ImageQuality::Balanced.value() { - 100 * 2 / 3 + Quality::Balanced } else if q == ImageQuality::Low.value() { - 100 / 2 + Quality::Low } else if q == ImageQuality::Best.value() { - 100 + Quality::Best } else { - (q >> 8 & 0xFF) * 2 + let mut b = (q >> 8 & 0xFF) * 2; + b = std::cmp::max(b, 10); + b = std::cmp::min(b, 200); + Quality::Custom(b as u32) } }; - let quality = Some((convert_quality(image_quality), hbb_common::get_time())); + let quality = Some((hbb_common::get_time(), convert_quality(image_quality))); if let Some(user) = self.users.get_mut(&id) { user.quality = quality; } else { @@ -254,43 +288,78 @@ impl VideoQoS { }, ); } - self.refresh(); + self.refresh(Some(RefreshType::SetImageQuality)); } pub fn user_network_delay(&mut self, id: i32, delay: u32) { - let mut refresh = true; let state = DelayState::from_delay(delay); + let debounce = 3; if let Some(user) = self.users.get_mut(&id) { - if let Some((old_state, old_delay, mut counter)) = user.delay { - let new_delay = (delay + old_delay) / 2; - let new_state = DelayState::from_delay(new_delay); - if old_state == new_state { - counter += 1; + if let Some(d) = &mut user.delay { + d.delay = (delay + d.delay) / 2; + let new_state = DelayState::from_delay(d.delay); + let slower_than_old_state = new_state as i32 - d.staging_state as i32; + let slower_than_old_state = if slower_than_old_state > 0 { + Some(true) + } else if slower_than_old_state < 0 { + Some(false) } else { - counter = 0; + None + }; + if d.slower_than_old_state == slower_than_old_state { + let old_counter = d.counter; + d.counter += delay / 1000 + 1; + if old_counter < debounce && d.counter >= debounce { + d.counter = 0; + d.state = d.staging_state; + d.staging_state = new_state; + } + if d.counter % debounce == 0 { + self.refresh(None); + } + } else { + d.counter = 0; + d.staging_state = new_state; + d.slower_than_old_state = slower_than_old_state; } - let debounce = 3; - refresh = counter == debounce; - user.delay = Some((new_state, new_delay, counter)); } else { - user.delay = Some((state, delay, 0)); + user.delay = Some(Delay { + state: DelayState::Normal, + staging_state: state, + delay, + counter: 0, + slower_than_old_state: None, + }); } } else { self.users.insert( id, UserData { - delay: Some((state, delay, 0)), + delay: Some(Delay { + state: DelayState::Normal, + staging_state: state, + delay, + counter: 0, + slower_than_old_state: None, + }), ..Default::default() }, ); } - if refresh { - self.refresh(); + } + + pub fn user_delay_response_elapsed(&mut self, id: i32, elapsed: u128) { + if let Some(user) = self.users.get_mut(&id) { + let old = user.response_delayed; + user.response_delayed = elapsed > 3000; + if old != user.response_delayed { + self.refresh(None); + } } } pub fn on_connection_close(&mut self, id: i32) { self.users.remove(&id); - self.refresh(); + self.refresh(None); } } diff --git a/src/server/video_service.rs b/src/server/video_service.rs index ad83e71d6..7c63acb68 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -514,13 +514,12 @@ fn run(sp: GenericService) -> ResultType<()> { let mut c = get_capturer(true, last_portable_service_running)?; let mut video_qos = VIDEO_QOS.lock().unwrap(); - video_qos.set_size(c.width as _, c.height as _); - video_qos.refresh(); - let mut spf = video_qos.spf(); - let bitrate = video_qos.bitrate(); + video_qos.refresh(None); + let mut spf; + let mut quality = video_qos.quality(); let abr = VideoQoS::abr_enabled(); drop(video_qos); - log::info!("init bitrate={}, abr enabled:{}", bitrate, abr); + log::info!("init quality={:?}, abr enabled:{}", quality, abr); let encoder_cfg = match Encoder::negotiated_codec() { scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => { @@ -528,7 +527,7 @@ fn run(sp: GenericService) -> ResultType<()> { name, width: c.width, height: c.height, - bitrate: bitrate as _, + quality, }) } name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => { @@ -536,7 +535,7 @@ fn run(sp: GenericService) -> ResultType<()> { width: c.width as _, height: c.height as _, timebase: [1, 1000], // Output timestamp precision - bitrate, + quality, codec: if name == scrap::CodecName::VP8 { VpxVideoCodecId::VP8 } else { @@ -548,7 +547,7 @@ fn run(sp: GenericService) -> ResultType<()> { scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig { width: c.width as _, height: c.height as _, - bitrate: bitrate as _, + quality, }), }; @@ -558,6 +557,7 @@ fn run(sp: GenericService) -> ResultType<()> { Err(err) => bail!("Failed to create encoder: {}", err), } c.set_use_yuv(encoder.use_yuv()); + VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate()); if *SWITCH.lock().unwrap() { log::debug!("Broadcasting display switch"); @@ -611,16 +611,12 @@ fn run(sp: GenericService) -> ResultType<()> { check_uac_switch(c.privacy_mode_id, c._capturer_privacy_mode_id)?; let mut video_qos = VIDEO_QOS.lock().unwrap(); - if video_qos.check_if_updated() { - log::debug!( - "qos is updated, target_bitrate:{}, fps:{}", - video_qos.bitrate(), - video_qos.fps() - ); - if video_qos.bitrate() > 0 { - allow_err!(encoder.set_bitrate(video_qos.bitrate())); - } - spf = video_qos.spf(); + spf = video_qos.spf(); + if quality != video_qos.quality() { + log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality()); + quality = video_qos.quality(); + allow_err!(encoder.set_quality(quality)); + video_qos.store_bitrate(encoder.bitrate()); } drop(video_qos);