codec set quality seperately and refactor network delay

Signed-off-by: 21pages <pages21@163.com>

libs/scrap/examples/benchmark.rs

@@ -2,7 +2,7 @@ use docopt::Docopt;
 use hbb_common::env_logger::{init_from_env, Env, DEFAULT_FILTER_ENV};
 use scrap::{
     aom::{AomDecoder, AomDecoderConfig, AomEncoder, AomEncoderConfig},
-    codec::{EncoderApi, EncoderCfg},
+    codec::{EncoderApi, EncoderCfg, Quality as Q},
     Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig,
     VpxVideoCodecId::{self, *},
     STRIDE_ALIGN,
@@ -15,13 +15,14 @@ const USAGE: &'static str = "
 Codec benchmark.
 
 Usage:
-  benchmark [--count=COUNT] [--bitrate=KBS] [--hw-pixfmt=PIXFMT]
+  benchmark [--count=COUNT] [--quality=QUALITY] [--hw-pixfmt=PIXFMT]
   benchmark (-h | --help)
 
 Options:
   -h --help             Show this screen.
   --count=COUNT         Capture frame count [default: 100].
-  --bitrate=KBS         Video bitrate in kilobits per second [default: 5000].
+  --quality=QUALITY     Video quality [default: Balanced].
+                        Valid values: Best, Balanced, Low.
   --hw-pixfmt=PIXFMT    Hardware codec pixfmt. [default: i420]
                         Valid values: i420, nv12.
 ";
@@ -29,7 +30,7 @@ Options:
 #[derive(Debug, serde::Deserialize)]
 struct Args {
     flag_count: usize,
-    flag_bitrate: usize,
+    flag_quality: Quality,
     flag_hw_pixfmt: Pixfmt,
 }
 
@@ -39,20 +40,32 @@ enum Pixfmt {
     NV12,
 }
 
+#[derive(Debug, serde::Deserialize)]
+enum Quality {
+    Best,
+    Balanced,
+    Low,
+}
+
 fn main() {
     init_from_env(Env::default().filter_or(DEFAULT_FILTER_ENV, "info"));
     let args: Args = Docopt::new(USAGE)
         .and_then(|d| d.deserialize())
         .unwrap_or_else(|e| e.exit());
-    let bitrate_k = args.flag_bitrate;
+    let quality = args.flag_quality;
     let yuv_count = args.flag_count;
     let (yuvs, width, height) = capture_yuv(yuv_count);
     println!(
-        "benchmark {}x{} bitrate:{}k hw_pixfmt:{:?}",
+        "benchmark {}x{} quality:{:?}k hw_pixfmt:{:?}",
-        width, height, bitrate_k, args.flag_hw_pixfmt
+        width, height, quality, args.flag_hw_pixfmt
     );
-    [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, bitrate_k, yuv_count));
+    let quality = match quality {
-    test_av1(&yuvs, width, height, bitrate_k, yuv_count);
+        Quality::Best => Q::Best,
+        Quality::Balanced => Q::Balanced,
+        Quality::Low => Q::Low,
+    };
+    [VP8, VP9].map(|c| test_vpx(c, &yuvs, width, height, quality, yuv_count));
+    test_av1(&yuvs, width, height, quality, yuv_count);
     #[cfg(feature = "hwcodec")]
     {
         use hwcodec::AVPixelFormat;
@@ -61,7 +74,7 @@ fn main() {
             Pixfmt::NV12 => AVPixelFormat::AV_PIX_FMT_NV12,
         };
         let yuvs = hw::vpx_yuv_to_hw_yuv(yuvs, width, height, hw_pixfmt);
-        hw::test(&yuvs, width, height, bitrate_k, yuv_count, hw_pixfmt);
+        hw::test(&yuvs, width, height, quality, yuv_count, hw_pixfmt);
     }
 }
 
@@ -95,14 +108,14 @@ fn test_vpx(
     yuvs: &Vec<Vec<u8>>,
     width: usize,
     height: usize,
-    bitrate_k: usize,
+    quality: Q,
     yuv_count: usize,
 ) {
     let config = EncoderCfg::VPX(VpxEncoderConfig {
         width: width as _,
         height: height as _,
         timebase: [1, 1000],
-        bitrate: bitrate_k as _,
+        quality,
         codec: codec_id,
         num_threads: (num_cpus::get() / 2) as _,
     });
@@ -148,11 +161,11 @@ fn test_vpx(
     );
 }
 
-fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, bitrate_k: usize, yuv_count: usize) {
+fn test_av1(yuvs: &Vec<Vec<u8>>, width: usize, height: usize, quality: Q, yuv_count: usize) {
     let config = EncoderCfg::AOM(AomEncoderConfig {
         width: width as _,
         height: height as _,
-        bitrate: bitrate_k as _,
+        quality,
     });
     let mut encoder = AomEncoder::new(config).unwrap();
     let start = Instant::now();
@@ -208,17 +221,18 @@ mod hw {
         yuvs: &Vec<Vec<u8>>,
         width: usize,
         height: usize,
-        bitrate_k: usize,
+        quality: Q,
         yuv_count: usize,
         pixfmt: AVPixelFormat,
     ) {
+        let bitrate = scrap::hwcodec::HwEncoder::convert_quality(quality);
         let ctx = EncodeContext {
             name: String::from(""),
             width: width as _,
             height: height as _,
             pixfmt,
             align: 0,
-            bitrate: (bitrate_k * 1000) as _,
+            bitrate: bitrate as i32 * 1000,
             timebase: [1, 30],
             gop: 60,
             quality: Quality_Default,

libs/scrap/examples/record-screen.rs

@@ -13,7 +13,7 @@ use std::time::{Duration, Instant};
 use std::{io, thread};
 
 use docopt::Docopt;
-use scrap::codec::{EncoderApi, EncoderCfg};
+use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q};
 use webm::mux;
 use webm::mux::Track;
 
@@ -24,17 +24,18 @@ const USAGE: &'static str = "
 Simple WebM screen capture.
 
 Usage:
-  record-screen <path> [--time=<s>] [--fps=<fps>] [--bv=<kbps>] [--ba=<kbps>] [--codec CODEC]
+  record-screen <path> [--time=<s>] [--fps=<fps>] [--quality=<quality>] [--ba=<kbps>] [--codec CODEC]
   record-screen (-h | --help)
 
 Options:
-  -h --help      Show this screen.
+  -h --help                 Show this screen.
-  --time=<s>     Recording duration in seconds.
+  --time=<s>                Recording duration in seconds.
-  --fps=<fps>    Frames per second [default: 30].
+  --fps=<fps>               Frames per second [default: 30].
-  --bv=<kbps>    Video bitrate in kilobits per second [default: 5000].
+  --quality=<quality>       Video quality [default: Balanced].
-  --ba=<kbps>    Audio bitrate in kilobits per second [default: 96].
+                            Valid values: Best, Balanced, Low.
-  --codec CODEC  Configure the codec used. [default: vp9]
+  --ba=<kbps>               Audio bitrate in kilobits per second [default: 96].
-                 Valid values: vp8, vp9.
+  --codec CODEC             Configure the codec used. [default: vp9]
+                            Valid values: vp8, vp9.
 ";
 
 #[derive(Debug, serde::Deserialize)]
@@ -43,7 +44,14 @@ struct Args {
     flag_codec: Codec,
     flag_time: Option<u64>,
     flag_fps: u64,
-    flag_bv: u32,
+    flag_quality: Quality,
+}
+
+#[derive(Debug, serde::Deserialize)]
+enum Quality {
+    Best,
+    Balanced,
+    Low,
 }
 
 #[derive(Debug, serde::Deserialize)]
@@ -97,12 +105,16 @@ fn main() -> io::Result<()> {
     let mut vt = webm.add_video_track(width, height, None, mux_codec);
 
     // Setup the encoder.
-
+    let quality = match args.flag_quality {
+        Quality::Best => Q::Best,
+        Quality::Balanced => Q::Balanced,
+        Quality::Low => Q::Low,
+    };
     let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig {
         width,
         height,
         timebase: [1, 1000],
-        bitrate: args.flag_bv,
+        quality,
         codec: vpx_codec,
         num_threads: 0,
     }))

libs/scrap/src/common/aom.rs

@@ -6,6 +6,7 @@
 
 include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs"));
 
+use crate::codec::{base_bitrate, Quality};
 use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN};
 use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result};
 use hbb_common::{
@@ -43,7 +44,7 @@ impl Default for aom_image_t {
 pub struct AomEncoderConfig {
     pub width: u32,
     pub height: u32,
-    pub bitrate: u32,
+    pub quality: Quality,
 }
 
 pub struct AomEncoder {
@@ -56,7 +57,6 @@ pub struct AomEncoder {
 mod webrtc {
     use super::*;
 
-    const kQpMin: u32 = 10;
     const kUsageProfile: u32 = AOM_USAGE_REALTIME;
     const kMinQindex: u32 = 145; // Min qindex threshold for QP scaling.
     const kMaxQindex: u32 = 205; // Max qindex threshold for QP scaling.
@@ -65,7 +65,8 @@ mod webrtc {
     const kRtpTicksPerSecond: i32 = 90000;
     const kMinimumFrameRate: f64 = 1.0;
 
-    const kQpMax: u32 = 25; // to-do: webrtc use dynamic value, no more than 63
+    pub const DEFAULT_Q_MAX: u32 = 56; // no more than 63
+    pub const DEFAULT_Q_MIN: u32 = 12; // no more than 63, litter than q_max
 
     fn number_of_threads(width: u32, height: u32, number_of_cores: usize) -> u32 {
         // Keep the number of encoder threads equal to the possible number of
@@ -78,7 +79,7 @@ mod webrtc {
         } else {
             // Use 2 threads for low res on ARM.
             #[cfg(any(target_arch = "arm", target_arch = "aarch64", target_os = "android"))]
-            if (width * height >= 320 * 180 && number_of_cores > 2) {
+            if width * height >= 320 * 180 && number_of_cores > 2 {
                 return 2;
             }
             // 1 thread less than VGA.
@@ -122,11 +123,23 @@ mod webrtc {
         c.g_threads = number_of_threads(cfg.width, cfg.height, num_cpus::get());
         c.g_timebase.num = 1;
         c.g_timebase.den = kRtpTicksPerSecond;
-        c.rc_target_bitrate = cfg.bitrate; // kilobits/sec.
         c.g_input_bit_depth = kBitDepth;
         c.kf_mode = aom_kf_mode::AOM_KF_DISABLED;
-        c.rc_min_quantizer = kQpMin;
+        let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality);
-        c.rc_max_quantizer = kQpMax;
+        if q_min > 0 && q_min < q_max && q_max < 64 {
+            c.rc_min_quantizer = q_min;
+            c.rc_max_quantizer = q_max;
+        } else {
+            c.rc_min_quantizer = DEFAULT_Q_MIN;
+            c.rc_max_quantizer = DEFAULT_Q_MAX;
+        }
+        let base_bitrate = base_bitrate(cfg.width as _, cfg.height as _);
+        let bitrate = base_bitrate * b / 100;
+        if bitrate > 0 {
+            c.rc_target_bitrate = bitrate;
+        } else {
+            c.rc_target_bitrate = base_bitrate;
+        }
         c.rc_undershoot_pct = 50;
         c.rc_overshoot_pct = 50;
         c.rc_buf_initial_sz = 600;
@@ -259,11 +272,24 @@ impl EncoderApi for AomEncoder {
         true
     }
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
+    fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
-        let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
+        let mut c = unsafe { *self.ctx.config.enc.to_owned() };
-        new_enc_cfg.rc_target_bitrate = bitrate;
+        let (q_min, q_max, b) = Self::convert_quality(quality);
-        call_aom!(aom_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
+        if q_min > 0 && q_min < q_max && q_max < 64 {
-        return Ok(());
+            c.rc_min_quantizer = q_min;
+            c.rc_max_quantizer = q_max;
+        }
+        let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
+        if bitrate > 0 {
+            c.rc_target_bitrate = bitrate;
+        }
+        call_aom!(aom_codec_enc_config_set(&mut self.ctx, &c));
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        let c = unsafe { *self.ctx.config.enc.to_owned() };
+        c.rc_target_bitrate
     }
 }
 
@@ -319,6 +345,35 @@ impl AomEncoder {
             ..Default::default()
         }
     }
+
+    pub fn convert_quality(quality: Quality) -> (u32, u32, u32) {
+        // we can use lower bitrate for av1
+        match quality {
+            Quality::Best => (12, 25, 100),
+            Quality::Balanced => (12, 35, 100 * 2 / 3),
+            Quality::Low => (18, 45, 50),
+            Quality::Custom(b) => {
+                let (q_min, q_max) = Self::calc_q_values(b);
+                (q_min, q_max, b)
+            }
+        }
+    }
+
+    #[inline]
+    fn calc_q_values(b: u32) -> (u32, u32) {
+        let b = std::cmp::min(b, 200);
+        let q_min1: i32 = 24;
+        let q_min2 = 12;
+        let q_max1 = 45;
+        let q_max2 = 25;
+
+        let t = b as f32 / 200.0;
+
+        let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
+        let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
+
+        (q_min, q_max)
+    }
 }
 
 impl Drop for AomEncoder {

libs/scrap/src/common/codec.rs

@@ -42,7 +42,7 @@ pub struct HwEncoderConfig {
     pub name: String,
     pub width: usize,
     pub height: usize,
-    pub bitrate: i32,
+    pub quality: Quality,
 }
 
 #[derive(Debug, Clone)]
@@ -61,7 +61,9 @@ pub trait EncoderApi {
 
     fn use_yuv(&self) -> bool;
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>;
+    fn set_quality(&mut self, quality: Quality) -> ResultType<()>;
+
+    fn bitrate(&self) -> u32;
 }
 
 pub struct Encoder {
@@ -471,3 +473,33 @@ fn enable_hwcodec_option() -> bool {
     }
     return true; // default is true
 }
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Quality {
+    Best,
+    Balanced,
+    Low,
+    Custom(u32),
+}
+
+impl Default for Quality {
+    fn default() -> Self {
+        Self::Balanced
+    }
+}
+
+pub fn base_bitrate(width: u32, height: u32) -> u32 {
+    #[allow(unused_mut)]
+    let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9
+    if base_bitrate == 0 {
+        base_bitrate = 1920 * 1080 / 1000;
+    }
+    #[cfg(target_os = "android")]
+    {
+        // fix when android screen shrinks
+        let fix = crate::Display::fix_quality() as u32;
+        log::debug!("Android screen, fix quality:{}", fix);
+        base_bitrate = base_bitrate * fix;
+    }
+    base_bitrate
+}

libs/scrap/src/common/hwcodec.rs

@@ -1,5 +1,5 @@
 use crate::{
-    codec::{EncoderApi, EncoderCfg},
+    codec::{base_bitrate, EncoderApi, EncoderCfg},
     hw, ImageFormat, ImageRgb, HW_STRIDE_ALIGN,
 };
 use hbb_common::{
@@ -34,6 +34,9 @@ pub struct HwEncoder {
     yuv: Vec<u8>,
     pub format: DataFormat,
     pub pixfmt: AVPixelFormat,
+    width: u32,
+    height: u32,
+    bitrate: u32, //kbs
 }
 
 impl EncoderApi for HwEncoder {
@@ -43,13 +46,19 @@ impl EncoderApi for HwEncoder {
     {
         match cfg {
             EncoderCfg::HW(config) => {
+                let b = Self::convert_quality(config.quality);
+                let base_bitrate = base_bitrate(config.width as _, config.height as _);
+                let mut bitrate = base_bitrate * b / 100;
+                if base_bitrate <= 0 {
+                    bitrate = base_bitrate;
+                }
                 let ctx = EncodeContext {
                     name: config.name.clone(),
                     width: config.width as _,
                     height: config.height as _,
                     pixfmt: DEFAULT_PIXFMT,
                     align: HW_STRIDE_ALIGN as _,
-                    bitrate: config.bitrate * 1000,
+                    bitrate: bitrate as i32 * 1000,
                     timebase: DEFAULT_TIME_BASE,
                     gop: DEFAULT_GOP,
                     quality: DEFAULT_HW_QUALITY,
@@ -70,6 +79,9 @@ impl EncoderApi for HwEncoder {
                         yuv: vec![],
                         format,
                         pixfmt: ctx.pixfmt,
+                        width: ctx.width as _,
+                        height: ctx.height as _,
+                        bitrate,
                     }),
                     Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
                 }
@@ -114,10 +126,19 @@ impl EncoderApi for HwEncoder {
         false
     }
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
+    fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
-        self.encoder.set_bitrate((bitrate * 1000) as _).ok();
+        let b = Self::convert_quality(quality);
+        let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
+        if bitrate > 0 {
+            self.encoder.set_bitrate((bitrate * 1000) as _).ok();
+            self.bitrate = bitrate;
+        }
         Ok(())
     }
+
+    fn bitrate(&self) -> u32 {
+        self.bitrate
+    }
 }
 
 impl HwEncoder {
@@ -159,6 +180,16 @@ impl HwEncoder {
             Err(_) => Ok(Vec::<EncodeFrame>::new()),
         }
     }
+
+    pub fn convert_quality(quality: crate::codec::Quality) -> u32 {
+        use crate::codec::Quality;
+        match quality {
+            Quality::Best => 150,
+            Quality::Balanced => 100,
+            Quality::Low => 50,
+            Quality::Custom(b) => b,
+        }
+    }
 }
 
 pub struct HwDecoder {

libs/scrap/src/common/vpxcodec.rs

@@ -7,7 +7,7 @@ use hbb_common::log;
 use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
 use hbb_common::ResultType;
 
-use crate::codec::EncoderApi;
+use crate::codec::{base_bitrate, EncoderApi, Quality};
 use crate::{GoogleImage, STRIDE_ALIGN};
 
 use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
@@ -19,6 +19,9 @@ use std::{ptr, slice};
 generate_call_macro!(call_vpx, false);
 generate_call_ptr_macro!(call_vpx_ptr);
 
+const DEFAULT_QP_MAX: u32 = 56; // no more than 63
+const DEFAULT_QP_MIN: u32 = 12; // no more than 63
+
 #[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
 pub enum VpxVideoCodecId {
     VP8,
@@ -64,8 +67,9 @@ impl EncoderApi for VpxEncoder {
                 c.g_h = config.height;
                 c.g_timebase.num = config.timebase[0];
                 c.g_timebase.den = config.timebase[1];
-                c.rc_target_bitrate = config.bitrate;
                 c.rc_undershoot_pct = 95;
+                // When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature.
+                // In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps.
                 c.rc_dropframe_thresh = 25;
                 c.g_threads = if config.num_threads == 0 {
                     num_cpus::get() as _
@@ -79,6 +83,21 @@ impl EncoderApi for VpxEncoder {
                 // c.kf_min_dist = 0;
                 // c.kf_max_dist = 999999;
                 c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot
+                let (q_min, q_max, b) = Self::convert_quality(config.quality);
+                if q_min > 0 && q_min < q_max && q_max < 64 {
+                    c.rc_min_quantizer = q_min;
+                    c.rc_max_quantizer = q_max;
+                } else {
+                    c.rc_min_quantizer = DEFAULT_QP_MIN;
+                    c.rc_max_quantizer = DEFAULT_QP_MAX;
+                }
+                let base_bitrate = base_bitrate(config.width as _, config.height as _);
+                let bitrate = base_bitrate * b / 100;
+                if bitrate > 0 {
+                    c.rc_target_bitrate = bitrate;
+                } else {
+                    c.rc_target_bitrate = base_bitrate;
+                }
 
                 /*
                 The VPX encoder supports two-pass encoding for rate control purposes.
@@ -177,11 +196,24 @@ impl EncoderApi for VpxEncoder {
         true
     }
 
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
+    fn set_quality(&mut self, quality: Quality) -> ResultType<()> {
-        let mut new_enc_cfg = unsafe { *self.ctx.config.enc.to_owned() };
+        let mut c = unsafe { *self.ctx.config.enc.to_owned() };
-        new_enc_cfg.rc_target_bitrate = bitrate;
+        let (q_min, q_max, b) = Self::convert_quality(quality);
-        call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
+        if q_min > 0 && q_min < q_max && q_max < 64 {
-        return Ok(());
+            c.rc_min_quantizer = q_min;
+            c.rc_max_quantizer = q_max;
+        }
+        let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
+        if bitrate > 0 {
+            c.rc_target_bitrate = bitrate;
+        }
+        call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &c));
+        Ok(())
+    }
+
+    fn bitrate(&self) -> u32 {
+        let c = unsafe { *self.ctx.config.enc.to_owned() };
+        c.rc_target_bitrate
     }
 }
 
@@ -258,6 +290,34 @@ impl VpxEncoder {
             ..Default::default()
         }
     }
+
+    fn convert_quality(quality: Quality) -> (u32, u32, u32) {
+        match quality {
+            Quality::Best => (6, 45, 150),
+            Quality::Balanced => (12, 56, 100 * 2 / 3),
+            Quality::Low => (18, 56, 50),
+            Quality::Custom(b) => {
+                let (q_min, q_max) = Self::calc_q_values(b);
+                (q_min, q_max, b)
+            }
+        }
+    }
+
+    #[inline]
+    fn calc_q_values(b: u32) -> (u32, u32) {
+        let b = std::cmp::min(b, 200);
+        let q_min1: i32 = 36;
+        let q_min2 = 12;
+        let q_max1 = 56;
+        let q_max2 = 37;
+
+        let t = b as f32 / 200.0;
+
+        let q_min: u32 = ((1.0 - t) * q_min1 as f32 + t * q_min2 as f32).round() as u32;
+        let q_max = ((1.0 - t) * q_max1 as f32 + t * q_max2 as f32).round() as u32;
+
+        (q_min, q_max)
+    }
 }
 
 impl Drop for VpxEncoder {
@@ -289,8 +349,8 @@ pub struct VpxEncoderConfig {
     pub height: c_uint,
     /// The timebase numerator and denominator (in seconds).
     pub timebase: [c_int; 2],
-    /// The target bitrate (in kilobits per second).
+    /// The image quality
-    pub bitrate: c_uint,
+    pub quality: Quality,
     /// The codec
     pub codec: VpxVideoCodecId,
     pub num_threads: u32,

src/server/connection.rs

@@ -196,6 +196,7 @@ pub struct Connection {
     #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
     tx_desktop_ready: mpsc::Sender<()>,
     closed: bool,
+    delay_response_instant: Instant,
 }
 
 impl ConnInner {
@@ -326,6 +327,7 @@ impl Connection {
             #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
             tx_desktop_ready: _tx_desktop_ready,
             closed: false,
+            delay_response_instant: Instant::now(),
         };
         if !conn.on_open(addr).await {
             conn.closed = true;
@@ -593,18 +595,19 @@ impl Connection {
                         break;
                     }
                     let time = get_time();
+                    let mut qos = video_service::VIDEO_QOS.lock().unwrap();
                     if time > 0 && conn.last_test_delay == 0 {
                         conn.last_test_delay = time;
                         let mut msg_out = Message::new();
-                        let qos = video_service::VIDEO_QOS.lock().unwrap();
                         msg_out.set_test_delay(TestDelay{
                             time,
                             last_delay:conn.network_delay.unwrap_or_default(),
-                            target_bitrate:qos.bitrate(),
+                            target_bitrate: qos.bitrate(),
                             ..Default::default()
                         });
                         conn.inner.send(msg_out.into());
                     }
+                    qos.user_delay_response_elapsed(conn.inner.id(), conn.delay_response_instant.elapsed().as_millis());
                 }
             }
         }
@@ -1188,7 +1191,9 @@ impl Connection {
     #[inline]
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) {
-        self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok();
+        self.tx_input
+            .send(MessageInput::Pointer((msg, conn_id)))
+            .ok();
     }
 
     #[inline]
@@ -1553,6 +1558,7 @@ impl Connection {
                     .unwrap()
                     .user_network_delay(self.inner.id(), new_delay);
                 self.network_delay = Some(new_delay);
+                self.delay_response_instant = Instant::now();
             }
         } else if let Some(message::Union::SwitchSidesResponse(_s)) = msg.union {
             #[cfg(feature = "flutter")]
@@ -1590,7 +1596,8 @@ impl Connection {
                         self.input_mouse(me, self.inner.id());
                     }
                 }
-                Some(message::Union::PointerDeviceEvent(pde)) => {
+                Some(message::Union::PointerDeviceEvent(pde)) =>
+                {
                     #[cfg(not(any(target_os = "android", target_os = "ios")))]
                     if self.peer_keyboard_enabled() {
                         MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);

src/server/video_qos.rs

@@ -1,4 +1,5 @@
 use super::*;
+use scrap::codec::Quality;
 use std::time::Duration;
 pub const FPS: u32 = 30;
 pub const MIN_FPS: u32 = 1;
@@ -18,21 +19,29 @@ impl Percent for ImageQuality {
     }
 }
 
-#[derive(Default, Debug)]
+#[derive(Default, Debug, Copy, Clone)]
+struct Delay {
+    state: DelayState,
+    staging_state: DelayState,
+    delay: u32,
+    counter: u32,
+    slower_than_old_state: Option<bool>,
+}
+
+#[derive(Default, Debug, Copy, Clone)]
 struct UserData {
     full_speed_fps: Option<u32>,
     custom_fps: Option<u32>,
-    quality: Option<(i32, i64)>,             // (quality, time)
+    quality: Option<(i64, Quality)>, // (time, quality)
-    delay: Option<(DelayState, u32, usize)>, // (state, ms, counter)
+    delay: Option<Delay>,
+    response_delayed: bool,
 }
 
 pub struct VideoQoS {
-    width: u32,
-    height: u32,
     fps: u32,
-    target_bitrate: u32,
+    quality: Quality,
-    updated: bool,
     users: HashMap<i32, UserData>,
+    bitrate_store: u32,
 }
 
 #[derive(PartialEq, Debug, Clone, Copy)]
@@ -43,6 +52,12 @@ enum DelayState {
     Broken = 1000,
 }
 
+impl Default for DelayState {
+    fn default() -> Self {
+        DelayState::Normal
+    }
+}
+
 impl DelayState {
     fn from_delay(delay: u32) -> Self {
         if delay > DelayState::Broken as u32 {
@@ -61,24 +76,19 @@ impl Default for VideoQoS {
     fn default() -> Self {
         VideoQoS {
             fps: FPS,
-            width: 0,
+            quality: Default::default(),
-            height: 0,
-            target_bitrate: 0,
-            updated: false,
             users: Default::default(),
+            bitrate_store: 0,
         }
     }
 }
 
-impl VideoQoS {
+#[derive(Debug, PartialEq, Eq)]
-    pub fn set_size(&mut self, width: u32, height: u32) {
+pub enum RefreshType {
-        if width == 0 || height == 0 {
+    SetImageQuality,
-            return;
+}
-        }
-        self.width = width;
-        self.height = height;
-    }
 
+impl VideoQoS {
     pub fn spf(&self) -> Duration {
         Duration::from_secs_f32(1. / (self.fps as f32))
     }
@@ -87,24 +97,23 @@ impl VideoQoS {
         self.fps
     }
 
-    pub fn bitrate(&self) -> u32 {
+    pub fn store_bitrate(&mut self, bitrate: u32) {
-        self.target_bitrate
+        self.bitrate_store = bitrate;
     }
 
-    pub fn check_if_updated(&mut self) -> bool {
+    pub fn bitrate(&self) -> u32 {
-        if self.updated {
+        self.bitrate_store
-            self.updated = false;
+    }
-            return true;
+
-        }
+    pub fn quality(&self) -> Quality {
-        return false;
+        self.quality
     }
 
     pub fn abr_enabled() -> bool {
         "N" != Config::get_option("enable-abr")
     }
 
-    pub fn refresh(&mut self) {
+    pub fn refresh(&mut self, typ: Option<RefreshType>) {
-        let mut updated = false;
         // fps
         let user_fps = |u: &UserData| {
             // full_speed_fps
@@ -117,13 +126,19 @@ impl VideoQoS {
             }
             // delay
             if let Some(delay) = u.delay {
-                fps = match delay.0 {
+                fps = match delay.state {
                     DelayState::Normal => fps,
                     DelayState::LowDelay => fps,
                     DelayState::HighDelay => fps / 2,
                     DelayState::Broken => fps / 4,
                 }
             }
+            // delay response
+            if u.response_delayed {
+                if fps > MIN_FPS + 2 {
+                    fps = MIN_FPS + 2;
+                }
+            }
             return fps;
         };
         let mut fps = self
@@ -136,64 +151,79 @@ impl VideoQoS {
         if fps > MAX_FPS {
             fps = MAX_FPS;
         }
-        if fps != self.fps {
+        self.fps = fps;
-            self.fps = fps;
-            updated = true;
-        }
 
         // quality
         // latest image quality
-        let latest = self
+        let latest_quality = self
             .users
             .iter()
-            // .map(|(_, u)| u.quality)
+            .map(|(_, u)| u.quality)
-            .filter(|u| u.1.quality != None)
+            .filter(|q| *q != None)
-            .max_by(|u1, u2| {
+            .max_by(|a, b| a.unwrap_or_default().0.cmp(&b.unwrap_or_default().0))
-                u1.1.quality
+            .unwrap_or_default()
-                    .unwrap_or_default()
+            .unwrap_or_default()
-                    .1
+            .1;
-                    .cmp(&u2.1.quality.unwrap_or_default().1)
+        let mut quality = latest_quality;
-            });
+
-        let quality = if let Some((id, data)) = latest {
+        // network delay
-            let mut quality = data.quality.unwrap_or_default().0;
+        if Self::abr_enabled() && typ != Some(RefreshType::SetImageQuality) {
-            if quality <= 0 {
+            // max delay
-                quality = ImageQuality::Balanced.as_percent() as _;
+            let delay = self
-            }
+                .users
-            // use latest's delay for quality
+                .iter()
-            if Self::abr_enabled() {
+                .map(|u| u.1.delay)
-                if let Some(Some((delay, _, _))) = self.users.get(id).map(|u| u.delay) {
+                .filter(|d| d.is_some())
-                    quality = match delay {
+                .max_by(|a, b| {
-                        DelayState::Normal => quality,
+                    (a.unwrap_or_default().state as u32).cmp(&(b.unwrap_or_default().state as u32))
-                        DelayState::LowDelay => std::cmp::min(quality, 50),
+                });
-                        DelayState::HighDelay => std::cmp::min(quality, 25),
+            let delay = delay.unwrap_or_default().unwrap_or_default().state;
-                        DelayState::Broken => 10,
+            if delay != DelayState::Normal {
-                    };
+                match self.quality {
+                    Quality::Best => {
+                        quality = Quality::Balanced;
+                    }
+                    Quality::Balanced => {
+                        quality = Quality::Low;
+                    }
+                    Quality::Low => {
+                        quality = Quality::Low;
+                    }
+                    Quality::Custom(b) => match delay {
+                        DelayState::LowDelay => {
+                            quality =
+                                Quality::Custom(if b >= 150 { 100 } else { std::cmp::min(50, b) });
+                        }
+                        DelayState::HighDelay => {
+                            quality =
+                                Quality::Custom(if b >= 100 { 50 } else { std::cmp::min(25, b) });
+                        }
+                        DelayState::Broken => {
+                            quality =
+                                Quality::Custom(if b >= 50 { 25 } else { std::cmp::min(10, b) });
+                        }
+                        DelayState::Normal => {}
+                    },
+                }
+            } else {
+                match self.quality {
+                    Quality::Low => {
+                        if latest_quality == Quality::Best {
+                            quality = Quality::Balanced;
+                        }
+                    }
+                    Quality::Custom(current_b) => {
+                        if let Quality::Custom(latest_b) = latest_quality {
+                            if current_b < latest_b / 2 {
+                                quality = Quality::Custom(latest_b / 2);
+                            }
+                        }
+                    }
+                    _ => {}
                 }
             }
-            quality
-        } else {
-            ImageQuality::Balanced.as_percent() as _
-        };
-        // bitrate
-        #[allow(unused_mut)]
-        let mut base_bitrate = ((self.width * self.height) / 800) as u32;
-        if base_bitrate == 0 {
-            base_bitrate = 1920 * 1080 / 800;
         }
-        #[cfg(target_os = "android")]
+        self.quality = quality;
-        {
-            // fix when android screen shrinks
-            let fix = scrap::Display::fix_quality() as u32;
-            log::debug!("Android screen, fix quality:{}", fix);
-            base_bitrate = base_bitrate * fix;
-        }
-        let target_bitrate = base_bitrate * quality as u32 / 100;
-        if self.target_bitrate != target_bitrate {
-            self.target_bitrate = target_bitrate;
-            updated = true;
-        }
-
-        self.updated = updated;
     }
 
     pub fn user_custom_fps(&mut self, id: i32, fps: u32) {
@@ -211,7 +241,7 @@ impl VideoQoS {
                 },
             );
         }
-        self.refresh();
+        self.refresh(None);
     }
 
     pub fn user_full_speed_fps(&mut self, id: i32, full_speed_fps: u32) {
@@ -226,23 +256,27 @@ impl VideoQoS {
                 },
             );
         }
-        self.refresh();
+        self.refresh(None);
     }
 
     pub fn user_image_quality(&mut self, id: i32, image_quality: i32) {
-        let convert_quality = |q: i32| -> i32 {
+        // https://github.com/rustdesk/rustdesk/blob/d716e2b40c38737f1aa3f16de0dec67394a6ac68/src/server/video_service.rs#L493
+        let convert_quality = |q: i32| {
             if q == ImageQuality::Balanced.value() {
-                100 * 2 / 3
+                Quality::Balanced
             } else if q == ImageQuality::Low.value() {
-                100 / 2
+                Quality::Low
             } else if q == ImageQuality::Best.value() {
-                100
+                Quality::Best
             } else {
-                (q >> 8 & 0xFF) * 2
+                let mut b = (q >> 8 & 0xFF) * 2;
+                b = std::cmp::max(b, 10);
+                b = std::cmp::min(b, 200);
+                Quality::Custom(b as u32)
             }
         };
 
-        let quality = Some((convert_quality(image_quality), hbb_common::get_time()));
+        let quality = Some((hbb_common::get_time(), convert_quality(image_quality)));
         if let Some(user) = self.users.get_mut(&id) {
             user.quality = quality;
         } else {
@@ -254,43 +288,78 @@ impl VideoQoS {
                 },
             );
         }
-        self.refresh();
+        self.refresh(Some(RefreshType::SetImageQuality));
     }
 
     pub fn user_network_delay(&mut self, id: i32, delay: u32) {
-        let mut refresh = true;
         let state = DelayState::from_delay(delay);
+        let debounce = 3;
         if let Some(user) = self.users.get_mut(&id) {
-            if let Some((old_state, old_delay, mut counter)) = user.delay {
+            if let Some(d) = &mut user.delay {
-                let new_delay = (delay + old_delay) / 2;
+                d.delay = (delay + d.delay) / 2;
-                let new_state = DelayState::from_delay(new_delay);
+                let new_state = DelayState::from_delay(d.delay);
-                if old_state == new_state {
+                let slower_than_old_state = new_state as i32 - d.staging_state as i32;
-                    counter += 1;
+                let slower_than_old_state = if slower_than_old_state > 0 {
+                    Some(true)
+                } else if slower_than_old_state < 0 {
+                    Some(false)
                 } else {
-                    counter = 0;
+                    None
+                };
+                if d.slower_than_old_state == slower_than_old_state {
+                    let old_counter = d.counter;
+                    d.counter += delay / 1000 + 1;
+                    if old_counter < debounce && d.counter >= debounce {
+                        d.counter = 0;
+                        d.state = d.staging_state;
+                        d.staging_state = new_state;
+                    }
+                    if d.counter % debounce == 0 {
+                        self.refresh(None);
+                    }
+                } else {
+                    d.counter = 0;
+                    d.staging_state = new_state;
+                    d.slower_than_old_state = slower_than_old_state;
                 }
-                let debounce = 3;
-                refresh = counter == debounce;
-                user.delay = Some((new_state, new_delay, counter));
             } else {
-                user.delay = Some((state, delay, 0));
+                user.delay = Some(Delay {
+                    state: DelayState::Normal,
+                    staging_state: state,
+                    delay,
+                    counter: 0,
+                    slower_than_old_state: None,
+                });
             }
         } else {
             self.users.insert(
                 id,
                 UserData {
-                    delay: Some((state, delay, 0)),
+                    delay: Some(Delay {
+                        state: DelayState::Normal,
+                        staging_state: state,
+                        delay,
+                        counter: 0,
+                        slower_than_old_state: None,
+                    }),
                     ..Default::default()
                 },
             );
         }
-        if refresh {
+    }
-            self.refresh();
+
+    pub fn user_delay_response_elapsed(&mut self, id: i32, elapsed: u128) {
+        if let Some(user) = self.users.get_mut(&id) {
+            let old = user.response_delayed;
+            user.response_delayed = elapsed > 3000;
+            if old != user.response_delayed {
+                self.refresh(None);
+            }
         }
     }
 
     pub fn on_connection_close(&mut self, id: i32) {
         self.users.remove(&id);
-        self.refresh();
+        self.refresh(None);
     }
 }

src/server/video_service.rs

@@ -514,13 +514,12 @@ fn run(sp: GenericService) -> ResultType<()> {
     let mut c = get_capturer(true, last_portable_service_running)?;
 
     let mut video_qos = VIDEO_QOS.lock().unwrap();
-    video_qos.set_size(c.width as _, c.height as _);
+    video_qos.refresh(None);
-    video_qos.refresh();
+    let mut spf;
-    let mut spf = video_qos.spf();
+    let mut quality = video_qos.quality();
-    let bitrate = video_qos.bitrate();
     let abr = VideoQoS::abr_enabled();
     drop(video_qos);
-    log::info!("init bitrate={}, abr enabled:{}", bitrate, abr);
+    log::info!("init quality={:?}, abr enabled:{}", quality, abr);
 
     let encoder_cfg = match Encoder::negotiated_codec() {
         scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => {
@@ -528,7 +527,7 @@ fn run(sp: GenericService) -> ResultType<()> {
                 name,
                 width: c.width,
                 height: c.height,
-                bitrate: bitrate as _,
+                quality,
             })
         }
         name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => {
@@ -536,7 +535,7 @@ fn run(sp: GenericService) -> ResultType<()> {
                 width: c.width as _,
                 height: c.height as _,
                 timebase: [1, 1000], // Output timestamp precision
-                bitrate,
+                quality,
                 codec: if name == scrap::CodecName::VP8 {
                     VpxVideoCodecId::VP8
                 } else {
@@ -548,7 +547,7 @@ fn run(sp: GenericService) -> ResultType<()> {
         scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig {
             width: c.width as _,
             height: c.height as _,
-            bitrate: bitrate as _,
+            quality,
         }),
     };
 
@@ -558,6 +557,7 @@ fn run(sp: GenericService) -> ResultType<()> {
         Err(err) => bail!("Failed to create encoder: {}", err),
     }
     c.set_use_yuv(encoder.use_yuv());
+    VIDEO_QOS.lock().unwrap().store_bitrate(encoder.bitrate());
 
     if *SWITCH.lock().unwrap() {
         log::debug!("Broadcasting display switch");
@@ -611,16 +611,12 @@ fn run(sp: GenericService) -> ResultType<()> {
         check_uac_switch(c.privacy_mode_id, c._capturer_privacy_mode_id)?;
 
         let mut video_qos = VIDEO_QOS.lock().unwrap();
-        if video_qos.check_if_updated() {
+        spf = video_qos.spf();
-            log::debug!(
+        if quality != video_qos.quality() {
-                "qos is updated, target_bitrate:{}, fps:{}",
+            log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality());
-                video_qos.bitrate(),
+            quality = video_qos.quality();
-                video_qos.fps()
+            allow_err!(encoder.set_quality(quality));
-            );
+            video_qos.store_bitrate(encoder.bitrate());
-            if video_qos.bitrate() > 0 {
-                allow_err!(encoder.set_bitrate(video_qos.bitrate()));
-            }
-            spf = video_qos.spf();
         }
         drop(video_qos);