hongshaorou/rustdesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #4 from Heap-Hop/hwcodec

Browse Source 
Update CodecFormat and refactor VideoQoS
This commit is contained in:
21pages 2022-07-06 10:39:20 +08:00 committed by GitHub
commit 61071a434d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 288 additions and 301 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
libs/hbb_common/protos/message.proto
View File

@ -457,18 +457,10 @@ message OptionMessage {
}
message TestDelay {
enum CodecFormat {
Unknown = 0;
VP8 = 1;
VP9 = 2;
H264 = 3;
H265 = 4;
}
int64 time = 1;
bool from_client = 2;
uint32 last_delay = 3;
uint32 target_bitrate = 4;
CodecFormat codec_format = 5;
}
message PublicKey {

4
libs/scrap/src/common/codec.rs
View File

@ -12,7 +12,7 @@ use crate::vpxcodec::*;
use hbb_common::{
anyhow::anyhow,
log,
message_proto::{test_delay, video_frame, EncodedVideoFrames, Message, VideoCodecState},
message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState},
ResultType,
};
#[cfg(feature = "hwcodec")]
@ -49,8 +49,6 @@ pub trait EncoderApi {
fn use_yuv(&self) -> bool;
fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()>;
fn get_codec_format(&self) -> test_delay::CodecFormat;
}
pub struct DecoderCfg {

9
libs/scrap/src/common/hwcodec.rs
View File

@ -6,7 +6,7 @@ use hbb_common::{
anyhow::{anyhow, Context},
config::HwCodecConfig,
lazy_static, log,
message_proto::{test_delay, EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame},
message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame},
ResultType,
};
use hwcodec::{
@ -120,13 +120,6 @@ impl EncoderApi for HwEncoder {
self.encoder.set_bitrate((bitrate * 1000) as _).ok();
Ok(())
}
fn get_codec_format(&self) -> test_delay::CodecFormat {
match self.format {
DataFormat::H264 => test_delay::CodecFormat::H264,
DataFormat::H265 => test_delay::CodecFormat::H265,
}
}
}
impl HwEncoder {

16
libs/scrap/src/common/vpxcodec.rs
View File

@ -3,9 +3,7 @@
// https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs
use hbb_common::anyhow::{anyhow, Context};
use hbb_common::message_proto::{
test_delay, EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame,
};
use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
use hbb_common::ResultType;
use crate::codec::EncoderApi;
@ -29,7 +27,6 @@ impl Default for VpxVideoCodecId {
pub struct VpxEncoder {
ctx: vpx_codec_ctx_t,
format: VpxVideoCodecId,
width: usize,
height: usize,
}
@ -99,17 +96,14 @@ impl EncoderApi for VpxEncoder {
{
match cfg {
crate::codec::EncoderCfg::VPX(config) => {
let format;
let i;
if cfg!(feature = "VP8") {
i = match config.codec {
VpxVideoCodecId::VP8 => call_vpx_ptr!(vpx_codec_vp8_cx()),
VpxVideoCodecId::VP9 => call_vpx_ptr!(vpx_codec_vp9_cx()),
};
format = config.codec;
} else {
i = call_vpx_ptr!(vpx_codec_vp9_cx());
format = VpxVideoCodecId::VP9;
}
let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
call_vpx!(vpx_codec_enc_config_default(i, &mut c, 0));
@ -196,7 +190,6 @@ impl EncoderApi for VpxEncoder {
Ok(Self {
ctx,
format,
width: config.width as _,
height: config.height as _,
})
@ -235,13 +228,6 @@ impl EncoderApi for VpxEncoder {
call_vpx!(vpx_codec_enc_config_set(&mut self.ctx, &new_enc_cfg));
return Ok(());
}
fn get_codec_format(&self) -> test_delay::CodecFormat {
match self.format {
VpxVideoCodecId::VP8 => test_delay::CodecFormat::VP8,
VpxVideoCodecId::VP9 => test_delay::CodecFormat::VP9,
}
}
}
impl VpxEncoder {

2
src/client.rs
View File

@ -40,7 +40,7 @@ pub use super::lang::*;
pub mod file_trait;
pub use file_trait::FileManager;
pub mod helper;
pub use helper::LatencyController;
pub use helper::*;
pub const SEC30: Duration = Duration::from_secs(30);
pub struct Client;

32
src/client/helper.rs
View File

@ -3,7 +3,7 @@ use std::{
time::Instant,
};
use hbb_common::log;
use hbb_common::{log, message_proto::{VideoFrame, video_frame}};
const MAX_LATENCY: i64 = 500;
const MIN_LATENCY: i64 = 100;
@ -57,3 +57,33 @@ impl LatencyController {
self.allow_audio
}
}
#[derive(PartialEq, Debug, Clone)]
pub enum CodecFormat {
VP9,
H264,
H265,
Unknown,
}
impl From<&VideoFrame> for CodecFormat {
fn from(it: &VideoFrame) -> Self {
match it.union {
Some(video_frame::Union::vp9s(_)) => CodecFormat::VP9,
Some(video_frame::Union::h264s(_)) => CodecFormat::H264,
Some(video_frame::Union::h265s(_)) => CodecFormat::H265,
_ => CodecFormat::Unknown,
}
}
}
impl ToString for CodecFormat {
fn to_string(&self) -> String {
match self {
CodecFormat::VP9 => "VP9".into(),
CodecFormat::H264 => "H264".into(),
CodecFormat::H265 => "H265".into(),
CodecFormat::Unknown => "Unknow".into(),
}
}
}

1
src/server.rs
View File

@ -38,6 +38,7 @@ pub const NAME_POS: &'static str = "";
mod connection;
mod service;
mod video_qos;
pub mod video_service;
use hbb_common::tcp::new_listener;

1
src/server/connection.rs
View File

@ -380,7 +380,6 @@ impl Connection {
time,
last_delay:qos.current_delay,
target_bitrate:qos.target_bitrate,
codec_format:qos.codec_format.into(),
..Default::default()
});
conn.inner.send(msg_out.into());

219
src/server/video_qos.rs Normal file
View File

@ -0,0 +1,219 @@
use super::*;
use std::time::Duration;
const FPS: u8 = 30;
trait Percent {
fn as_percent(&self) -> u32;
}
impl Percent for ImageQuality {
fn as_percent(&self) -> u32 {
match self {
ImageQuality::NotSet => 0,
ImageQuality::Low => 50,
ImageQuality::Balanced => 66,
ImageQuality::Best => 100,
}
}
}
pub struct VideoQoS {
width: u32,
height: u32,
user_image_quality: u32,
current_image_quality: u32,
enable_abr: bool,
pub current_delay: u32,
pub fps: u8, // abr
pub target_bitrate: u32, // abr
updated: bool,
state: DelayState,
debounce_count: u32,
}
#[derive(PartialEq, Debug)]
enum DelayState {
Normal = 0,
LowDelay = 200,
HighDelay = 500,
Broken = 1000,
}
impl DelayState {
fn from_delay(delay: u32) -> Self {
if delay > DelayState::Broken as u32 {
DelayState::Broken
} else if delay > DelayState::HighDelay as u32 {
DelayState::HighDelay
} else if delay > DelayState::LowDelay as u32 {
DelayState::LowDelay
} else {
DelayState::Normal
}
}
}
impl Default for VideoQoS {
fn default() -> Self {
VideoQoS {
fps: FPS,
user_image_quality: ImageQuality::Balanced.as_percent(),
current_image_quality: ImageQuality::Balanced.as_percent(),
enable_abr: false,
width: 0,
height: 0,
current_delay: 0,
target_bitrate: 0,
updated: false,
state: DelayState::Normal,
debounce_count: 0,
}
}
}
impl VideoQoS {
pub fn set_size(&mut self, width: u32, height: u32) {
if width == 0 || height == 0 {
return;
}
self.width = width;
self.height = height;
}
pub fn spf(&mut self) -> Duration {
if self.fps <= 0 {
self.fps = FPS;
}
Duration::from_secs_f32(1. / (self.fps as f32))
}
// update_network_delay periodically
// decrease the bitrate when the delay gets bigger
pub fn update_network_delay(&mut self, delay: u32) {
if self.current_delay.eq(&0) {
self.current_delay = delay;
return;
}
self.current_delay = delay / 2 + self.current_delay / 2;
log::trace!(
"VideoQoS update_network_delay:{}, {}, state:{:?}",
self.current_delay,
delay,
self.state,
);
// ABR
if !self.enable_abr {
return;
}
let current_state = DelayState::from_delay(self.current_delay);
if current_state != self.state && self.debounce_count > 5 {
log::debug!(
"VideoQoS state changed:{:?} -> {:?}",
self.state,
current_state
);
self.state = current_state;
self.debounce_count = 0;
self.refresh_quality();
} else {
self.debounce_count += 1;
}
}
fn refresh_quality(&mut self) {
match self.state {
DelayState::Normal => {
self.fps = FPS;
self.current_image_quality = self.user_image_quality;
}
DelayState::LowDelay => {
self.fps = FPS;
self.current_image_quality = std::cmp::min(self.user_image_quality, 50);
}
DelayState::HighDelay => {
self.fps = FPS / 2;
self.current_image_quality = std::cmp::min(self.user_image_quality, 25);
}
DelayState::Broken => {
self.fps = FPS / 4;
self.current_image_quality = 10;
}
}
let _ = self.generate_bitrate().ok();
self.updated = true;
}
// handle image_quality change from peer
pub fn update_image_quality(&mut self, image_quality: i32) {
let image_quality = Self::convert_quality(image_quality) as _;
log::debug!("VideoQoS update_image_quality: {}", image_quality);
if self.current_image_quality != image_quality {
self.current_image_quality = image_quality;
let _ = self.generate_bitrate().ok();
self.updated = true;
}
self.user_image_quality = self.current_image_quality;
}
pub fn generate_bitrate(&mut self) -> ResultType<u32> {
// https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/
if self.width == 0 || self.height == 0 {
bail!("Fail to generate_bitrate, width or height is not set");
}
if self.current_image_quality == 0 {
self.current_image_quality = ImageQuality::Balanced.as_percent();
}
let base_bitrate = ((self.width * self.height) / 800) as u32;
#[cfg(target_os = "android")]
{
// fix when andorid screen shrinks
let fix = Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
let base_bitrate = base_bitrate * fix;
self.target_bitrate = base_bitrate * self.current_image_quality / 100;
Ok(self.target_bitrate)
}
#[cfg(not(target_os = "android"))]
{
self.target_bitrate = base_bitrate * self.current_image_quality / 100;
Ok(self.target_bitrate)
}
}
pub fn check_if_updated(&mut self) -> bool {
if self.updated {
self.updated = false;
return true;
}
return false;
}
pub fn reset(&mut self) {
*self = Default::default();
}
pub fn check_abr_config(&mut self) -> bool {
self.enable_abr = if let Some(v) = Config2::get().options.get("enable-abr") {
v != "N"
} else {
true // default is true
};
self.enable_abr
}
pub fn convert_quality(q: i32) -> i32 {
if q == ImageQuality::Balanced.value() {
100 * 2 / 3
} else if q == ImageQuality::Low.value() {
100 / 2
} else if q == ImageQuality::Best.value() {
100
} else {
(q >> 8 & 0xFF) * 2
}
}
}

227
src/server/video_service.rs
View File

@ -18,7 +18,7 @@
// to-do:
// https://slhck.info/video/2017/03/01/rate-control.html
use super::*;
use super::{video_qos::VideoQoS, *};
use hbb_common::tokio::sync::{
mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
Mutex as TokioMutex,
@ -37,7 +37,6 @@ use std::{
use virtual_display;
pub const NAME: &'static str = "video";
const FPS: u8 = 30;
lazy_static::lazy_static! {
static ref CURRENT_DISPLAY: Arc<Mutex<usize>> = Arc::new(Mutex::new(usize::MAX));
@ -52,225 +51,6 @@ lazy_static::lazy_static! {
pub static ref VIDEO_QOS: Arc<Mutex<VideoQoS>> = Default::default();
}
trait Percent {
fn as_percent(&self) -> u32;
}
impl Percent for ImageQuality {
fn as_percent(&self) -> u32 {
match self {
ImageQuality::NotSet => 0,
ImageQuality::Low => 50,
ImageQuality::Balanced => 66,
ImageQuality::Best => 100,
}
}
}
pub struct VideoQoS {
width: u32,
height: u32,
user_image_quality: u32,
current_image_quality: u32,
enable_abr: bool,
pub codec_format: test_delay::CodecFormat,
pub current_delay: u32,
pub fps: u8, // abr
pub target_bitrate: u32, // abr
updated: bool,
state: DelayState,
debounce_count: u32,
}
#[derive(PartialEq, Debug)]
enum DelayState {
Normal = 0,
LowDelay = 200,
HighDelay = 500,
Broken = 1000,
}
impl DelayState {
fn from_delay(delay: u32) -> Self {
if delay > DelayState::Broken as u32 {
DelayState::Broken
} else if delay > DelayState::HighDelay as u32 {
DelayState::HighDelay
} else if delay > DelayState::LowDelay as u32 {
DelayState::LowDelay
} else {
DelayState::Normal
}
}
}
impl Default for VideoQoS {
fn default() -> Self {
VideoQoS {
fps: FPS,
user_image_quality: ImageQuality::Balanced.as_percent(),
current_image_quality: ImageQuality::Balanced.as_percent(),
enable_abr: false,
codec_format: Default::default(),
width: 0,
height: 0,
current_delay: 0,
target_bitrate: 0,
updated: false,
state: DelayState::Normal,
debounce_count: 0,
}
}
}
impl VideoQoS {
pub fn set_size(&mut self, width: u32, height: u32) {
if width == 0 || height == 0 {
return;
}
self.width = width;
self.height = height;
}
pub fn spf(&mut self) -> Duration {
if self.fps <= 0 {
self.fps = FPS;
}
time::Duration::from_secs_f32(1. / (self.fps as f32))
}
// update_network_delay periodically
// decrease the bitrate when the delay gets bigger
pub fn update_network_delay(&mut self, delay: u32) {
if self.current_delay.eq(&0) {
self.current_delay = delay;
return;
}
self.current_delay = delay / 2 + self.current_delay / 2;
log::trace!(
"VideoQoS update_network_delay:{}, {}, state:{:?}",
self.current_delay,
delay,
self.state,
);
// ABR
if !self.enable_abr {
return;
}
let current_state = DelayState::from_delay(self.current_delay);
if current_state != self.state && self.debounce_count > 5 {
log::debug!(
"VideoQoS state changed:{:?} -> {:?}",
self.state,
current_state
);
self.state = current_state;
self.debounce_count = 0;
self.refresh_quality();
} else {
self.debounce_count += 1;
}
}
fn refresh_quality(&mut self) {
match self.state {
DelayState::Normal => {
self.fps = FPS;
self.current_image_quality = self.user_image_quality;
}
DelayState::LowDelay => {
self.fps = FPS;
self.current_image_quality = std::cmp::min(self.user_image_quality, 50);
}
DelayState::HighDelay => {
self.fps = FPS / 2;
self.current_image_quality = std::cmp::min(self.user_image_quality, 25);
}
DelayState::Broken => {
self.fps = FPS / 4;
self.current_image_quality = 10;
}
}
let _ = self.generate_bitrate().ok();
self.updated = true;
}
// handle image_quality change from peer
pub fn update_image_quality(&mut self, image_quality: i32) {
let image_quality = Self::convert_quality(image_quality) as _;
log::debug!("VideoQoS update_image_quality: {}", image_quality);
if self.current_image_quality != image_quality {
self.current_image_quality = image_quality;
let _ = self.generate_bitrate().ok();
self.updated = true;
}
self.user_image_quality = self.current_image_quality;
}
pub fn generate_bitrate(&mut self) -> ResultType<u32> {
// https://www.nvidia.com/en-us/geforce/guides/broadcasting-guide/
if self.width == 0 || self.height == 0 {
bail!("Fail to generate_bitrate, width or height is not set");
}
if self.current_image_quality == 0 {
self.current_image_quality = ImageQuality::Balanced.as_percent();
}
let base_bitrate = ((self.width * self.height) / 800) as u32;
#[cfg(target_os = "android")]
{
// fix when andorid screen shrinks
let fix = Display::fix_quality() as u32;
log::debug!("Android screen, fix quality:{}", fix);
let base_bitrate = base_bitrate * fix;
self.target_bitrate = base_bitrate * self.current_image_quality / 100;
Ok(self.target_bitrate)
}
#[cfg(not(target_os = "android"))]
{
self.target_bitrate = base_bitrate * self.current_image_quality / 100;
Ok(self.target_bitrate)
}
}
pub fn check_if_updated(&mut self) -> bool {
if self.updated {
self.updated = false;
return true;
}
return false;
}
pub fn reset(&mut self) {
*self = Default::default();
}
fn check_abr_config(&mut self) -> bool {
self.enable_abr = if let Some(v) = Config2::get().options.get("enable-abr") {
v != "N"
} else {
true // default is true
};
self.enable_abr
}
pub fn convert_quality(q: i32) -> i32 {
if q == ImageQuality::Balanced.value() {
100 * 2 / 3
} else if q == ImageQuality::Low.value() {
100 / 2
} else if q == ImageQuality::Best.value() {
100
} else {
(q >> 8 & 0xFF) * 2
}
}
}
fn is_capturer_mag_supported() -> bool {
#[cfg(windows)]
return scrap::CapturerMag::is_supported();
@ -564,7 +344,7 @@ fn run(sp: GenericService) -> ResultType<()> {
let mut spf = video_qos.spf();
let bitrate = video_qos.generate_bitrate()?;
let abr = video_qos.check_abr_config();
drop(video_qos);
log::info!("init bitrate={}, abr enabled:{}", bitrate, abr);
let encoder_cfg = match Encoder::current_hw_encoder_name() {
@ -590,9 +370,6 @@ fn run(sp: GenericService) -> ResultType<()> {
Err(err) => bail!("Failed to create encoder: {}", err),
}
video_qos.codec_format = encoder.get_codec_format();
drop(video_qos);
let privacy_mode_id = *PRIVACY_MODE_CONN_ID.lock().unwrap();
#[cfg(not(windows))]
let captuerer_privacy_mode_id = privacy_mode_id;

62
src/ui/remote.rs
View File

@ -234,25 +234,13 @@ impl sciter::EventHandler for Handler {
}
}
#[derive(Debug)]
#[derive(Debug, Default)]
struct QualityStatus {
speed: String,
fps: i32,
delay: i32,
target_bitrate: i32,
codec_format: test_delay::CodecFormat,
}
impl Default for QualityStatus {
fn default() -> Self {
Self {
speed: Default::default(),
fps: -1,
delay: -1,
target_bitrate: -1,
codec_format: test_delay::CodecFormat::Unknown,
}
}
speed: Option<String>,
fps: Option<i32>,
delay: Option<i32>,
target_bitrate: Option<i32>,
codec_format: Option<CodecFormat>,
}
impl Handler {
@ -271,21 +259,16 @@ impl Handler {
}
fn update_quality_status(&self, status: QualityStatus) {
let codec_format = match status.codec_format {
test_delay::CodecFormat::Unknown => "Unknown",
test_delay::CodecFormat::VP8 => "VP8",
test_delay::CodecFormat::VP9 => "VP9",
test_delay::CodecFormat::H264 => "H264",
test_delay::CodecFormat::H265 => "H265",
};
self.call2(
"updateQualityStatus",
&make_args!(
status.speed,
status.fps,
status.delay,
status.target_bitrate,
codec_format
status.speed.map_or(Value::null(), |it| it.into()),
status.fps.map_or(Value::null(), |it| it.into()),
status.delay.map_or(Value::null(), |it| it.into()),
status.target_bitrate.map_or(Value::null(), |it| it.into()),
status
.codec_format
.map_or(Value::null(), |it| it.to_string().into())
),
);
}
@ -1365,6 +1348,7 @@ async fn io_loop(handler: Handler) {
clipboard_file_context: None,
data_count: Arc::new(AtomicUsize::new(0)),
frame_count,
video_format: CodecFormat::Unknown,
};
remote.io_loop(&key, &token).await;
remote.sync_jobs_status_to_local().await;
@ -1417,6 +1401,7 @@ struct Remote {
clipboard_file_context: Option<Box<CliprdrClientContext>>,
data_count: Arc<AtomicUsize>,
frame_count: Arc<AtomicUsize>,
video_format: CodecFormat,
}
impl Remote {
@ -1506,8 +1491,8 @@ impl Remote {
let speed = format!("{:.2}kB/s", speed as f32 / 1024 as f32);
let fps = self.frame_count.swap(0, Ordering::Relaxed) as _;
self.handler.update_quality_status(QualityStatus {
speed,
fps,
speed:Some(speed),
fps:Some(fps),
..Default::default()
});
}
@ -2038,6 +2023,14 @@ impl Remote {
self.handler.call2("closeSuccess", &make_args!());
self.handler.call("adaptSize", &make_args!());
}
let incomming_format = CodecFormat::from(&vf);
if self.video_format != incomming_format {
self.video_format = incomming_format.clone();
self.handler.update_quality_status(QualityStatus {
codec_format: Some(incomming_format),
..Default::default()
})
};
self.video_sender.send(MediaData::VideoFrame(vf)).ok();
}
Some(message::Union::hash(hash)) => {
@ -2612,9 +2605,8 @@ impl Interface for Handler {
async fn handle_test_delay(&mut self, t: TestDelay, peer: &mut Stream) {
if !t.from_client {
self.update_quality_status(QualityStatus {
delay: t.last_delay as _,
target_bitrate: t.target_bitrate as _,
codec_format: t.codec_format.enum_value_or_default(),
delay: Some(t.last_delay as _),
target_bitrate: Some(t.target_bitrate as _),
..Default::default()
});
handle_test_delay(t, peer).await;

8
src/ui/remote.tis
View File

@ -492,10 +492,10 @@ class QualityMonitor: Reactor.Component
$(#quality-monitor).content(<QualityMonitor />);
handler.updateQualityStatus = function(speed, fps, delay, bitrate, codec_format) {
speed ? qualityMonitorData[0] = speed:null;
fps > -1 ? qualityMonitorData[1] = fps:null;
delay > -1 ? qualityMonitorData[2] = delay:null;
bitrate > -1 ? qualityMonitorData[3] = bitrate:null;
codec_format != "Unknown" ? qualityMonitorData[4] = codec_format:null;
fps ? qualityMonitorData[1] = fps:null;
delay ? qualityMonitorData[2] = delay:null;
bitrate ? qualityMonitorData[3] = bitrate:null;
codec_format ? qualityMonitorData[4] = codec_format:null;
qualityMonitor.update();
}