hongshaorou/rustdesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix multi display fps control (#8455)

Browse Source 
* Calculate fps without distinguish displays, use one fps control
  because the controlled side control fps of all displays with one FPS
  variable.
* Because all displays decode frame in one thread, when there are N
  displays, the video frames received in one second is `fps * N`, so the
  calculated decode fps should be divided by N. Because the actual
  display count is not obvious in rust, when no data frame is received for 5 seconds, the display is considered inactive, and only the active display is used as the dividend.

Signed-off-by: 21pages <sunboeasy@gmail.com>
This commit is contained in:
21pages 2024-06-24 19:41:15 +08:00 committed by GitHub
parent 65edd55516
commit 1765c7bbf4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 133 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

81
src/client.rs
View File

@ -43,8 +43,7 @@ use hbb_common::{
rand,
rendezvous_proto::*,
socket_client,
sodiumoxide::base64,
sodiumoxide::crypto::sign,
sodiumoxide::{base64, crypto::sign},
tcp::FramedStream,
timeout,
tokio::time::Duration,
@ -2093,8 +2092,6 @@ pub type MediaSender = mpsc::Sender<MediaData>;
struct VideoHandlerController {
handler: VideoHandler,
count: u128,
duration: std::time::Duration,
skip_beginning: u32,
}
@ -2111,7 +2108,7 @@ pub fn start_video_audio_threads<F, T>(
MediaSender,
MediaSender,
Arc<RwLock<HashMap<usize, ArrayQueue<VideoFrame>>>>,
Arc<RwLock<HashMap<usize, usize>>>,
Arc<RwLock<Option<usize>>>,
Arc<RwLock<Option<Chroma>>>,
)
where
@ -2123,8 +2120,8 @@ where
let video_queue_map_cloned = video_queue_map.clone();
let mut video_callback = video_callback;
let fps_map = Arc::new(RwLock::new(HashMap::new()));
let decode_fps_map = fps_map.clone();
let fps = Arc::new(RwLock::new(None));
let decode_fps_map = fps.clone();
let chroma = Arc::new(RwLock::new(None));
let chroma_cloned = chroma.clone();
let mut last_chroma = None;
@ -2134,9 +2131,8 @@ where
sync_cpu_usage();
get_hwcodec_config();
let mut handler_controller_map = HashMap::new();
// let mut count = Vec::new();
// let mut duration = std::time::Duration::ZERO;
// let mut skip_beginning = Vec::new();
let mut count = 0;
let mut duration = std::time::Duration::ZERO;
loop {
if let Ok(data) = video_receiver.recv() {
match data {
@ -2169,8 +2165,6 @@ where
display,
VideoHandlerController {
handler: VideoHandler::new(format, display),
count: 0,
duration: std::time::Duration::ZERO,
skip_beginning: 0,
},
);
@ -2178,6 +2172,8 @@ where
if let Some(handler_controller) = handler_controller_map.get_mut(&display) {
let mut pixelbuffer = true;
let mut tmp_chroma = None;
let format_changed =
handler_controller.handler.decoder.format() != format;
match handler_controller.handler.handle_frame(
vf,
&mut pixelbuffer,
@ -2198,27 +2194,14 @@ where
}
// fps calculation
// The first frame will be very slow
if handler_controller.skip_beginning < 5 {
handler_controller.skip_beginning += 1;
continue;
}
handler_controller.duration += start.elapsed();
handler_controller.count += 1;
if handler_controller.count % 10 == 0 {
fps_map.write().unwrap().insert(
display,
(handler_controller.count * 1000
/ handler_controller.duration.as_millis())
as usize,
);
}
// Clear to get real-time fps
if handler_controller.count > 150 {
handler_controller.count = 0;
handler_controller.duration = Duration::ZERO;
}
fps_calculate(
handler_controller,
&fps,
format_changed,
start.elapsed(),
&mut count,
&mut duration,
);
}
Err(e) => {
// This is a simple workaround.
@ -2334,6 +2317,38 @@ pub fn start_audio_thread() -> MediaSender {
audio_sender
}
#[inline]
fn fps_calculate(
handler_controller: &mut VideoHandlerController,
fps: &Arc<RwLock<Option<usize>>>,
format_changed: bool,
elapsed: std::time::Duration,
count: &mut usize,
duration: &mut std::time::Duration,
) {
if format_changed {
*count = 0;
*duration = std::time::Duration::ZERO;
handler_controller.skip_beginning = 0;
}
// // The first frame will be very slow
if handler_controller.skip_beginning < 3 {
handler_controller.skip_beginning += 1;
return;
}
*duration += elapsed;
*count += 1;
let ms = duration.as_millis();
if *count % 10 == 0 && ms > 0 {
*fps.write().unwrap() = Some((*count as usize) * 1000 / (ms as usize));
}
// Clear to get real-time fps
if *count >= 30 {
*count = 0;
*duration = Duration::ZERO;
}
}
fn get_hwcodec_config() {
// for sciter and unilink
#[cfg(feature = "hwcodec")]

146
src/client/io_loop.rs
View File

@ -71,8 +71,8 @@ pub struct Remote<T: InvokeUiSession> {
frame_count_map: Arc<RwLock<HashMap<usize, usize>>>,
video_format: CodecFormat,
elevation_requested: bool,
fps_control_map: HashMap<usize, FpsControl>,
decode_fps_map: Arc<RwLock<HashMap<usize, usize>>>,
fps_control: FpsControl,
decode_fps: Arc<RwLock<Option<usize>>>,
chroma: Arc<RwLock<Option<Chroma>>>,
}
@ -85,7 +85,7 @@ impl<T: InvokeUiSession> Remote<T> {
receiver: mpsc::UnboundedReceiver<Data>,
sender: mpsc::UnboundedSender<Data>,
frame_count_map: Arc<RwLock<HashMap<usize, usize>>>,
decode_fps: Arc<RwLock<HashMap<usize, usize>>>,
decode_fps: Arc<RwLock<Option<usize>>>,
chroma: Arc<RwLock<Option<Chroma>>>,
) -> Self {
Self {
@ -110,8 +110,8 @@ impl<T: InvokeUiSession> Remote<T> {
stop_voice_call_sender: None,
voice_call_request_timestamp: None,
elevation_requested: false,
fps_control_map: Default::default(),
decode_fps_map: decode_fps,
fps_control: Default::default(),
decode_fps,
chroma,
}
}
@ -971,69 +971,84 @@ impl<T: InvokeUiSession> Remote<T> {
if custom_fps < 5 || custom_fps > 120 {
custom_fps = 30;
}
let decode_fps_read = self.decode_fps_map.read().unwrap();
for (display, decode_fps) in decode_fps_read.iter() {
let video_queue_map_read = self.video_queue_map.read().unwrap();
let Some(video_queue) = video_queue_map_read.get(display) else {
continue;
};
if !self.fps_control_map.contains_key(display) {
self.fps_control_map.insert(*display, FpsControl::default());
let ctl = &mut self.fps_control;
let len = self
.video_queue_map
.read()
.unwrap()
.iter()
.map(|v| v.1.len())
.max()
.unwrap_or_default();
let decode_fps = self.decode_fps.read().unwrap().clone();
let Some(mut decode_fps) = decode_fps else {
return;
};
if cfg!(feature = "flutter") {
let active_displays = ctl
.last_active_time
.iter()
.filter(|t| t.1.elapsed().as_secs() < 5)
.count();
if active_displays > 1 {
decode_fps = decode_fps / active_displays;
}
let Some(ctl) = self.fps_control_map.get_mut(display) else {
return;
};
}
let mut limited_fps = if direct {
decode_fps * 9 / 10 // 30 got 27
} else {
decode_fps * 4 / 5 // 30 got 24
};
if limited_fps > custom_fps {
limited_fps = custom_fps;
}
let should_decrease = (len > 1
&& ctl.last_auto_fps.clone().unwrap_or(custom_fps as _) > limited_fps)
|| len > std::cmp::max(1, limited_fps / 2);
let len = video_queue.len();
let decode_fps = *decode_fps;
let mut limited_fps = if direct {
decode_fps * 9 / 10 // 30 got 27
} else {
decode_fps * 4 / 5 // 30 got 24
};
if limited_fps > custom_fps {
limited_fps = custom_fps;
}
let should_decrease = len > 1 && ctl.last_auto_fps.unwrap_or(0) > limited_fps as i32;
// increase judgement
if len <= 1 {
// increase judgement
if len <= 1 {
if ctl.idle_counter < usize::MAX {
ctl.idle_counter += 1;
} else {
ctl.idle_counter = 0;
}
let mut should_increase = false;
if let Some(last_auto_fps) = ctl.last_auto_fps {
// ever set
if last_auto_fps + 3 <= limited_fps as i32 && ctl.idle_counter > 3 {
// limited_fps is 5 larger than last set, and idle time is more than 3 seconds
should_increase = true;
}
} else {
ctl.idle_counter = 0;
}
let mut should_increase = false;
if let Some(last_auto_fps) = ctl.last_auto_fps.clone() {
// ever set
if last_auto_fps + 3 <= limited_fps && ctl.idle_counter > 3 {
// limited_fps is 3 larger than last set, and idle time is more than 3 seconds
should_increase = true;
}
if ctl.last_auto_fps.is_none() || should_decrease || should_increase {
// limited_fps to ensure decoding is faster than encoding
let mut auto_fps = limited_fps as i32;
if auto_fps < 1 {
auto_fps = 1;
}
// send custom fps
let mut misc = Misc::new();
misc.set_option(OptionMessage {
custom_fps: auto_fps,
..Default::default()
});
let mut msg = Message::new();
msg.set_misc(misc);
self.sender.send(Data::Message(msg)).ok();
ctl.last_queue_size = len;
ctl.last_auto_fps = Some(auto_fps);
}
if ctl.last_auto_fps.is_none() || should_decrease || should_increase {
// limited_fps to ensure decoding is faster than encoding
let mut auto_fps = limited_fps;
if should_decrease && limited_fps < len {
auto_fps = limited_fps / 2;
}
// send refresh
if auto_fps < 1 {
auto_fps = 1;
}
let mut misc = Misc::new();
misc.set_option(OptionMessage {
custom_fps: auto_fps as _,
..Default::default()
});
let mut msg = Message::new();
msg.set_misc(misc);
self.sender.send(Data::Message(msg)).ok();
log::info!("Set fps to {}", auto_fps);
ctl.last_queue_size = len;
ctl.last_auto_fps = Some(auto_fps);
}
// send refresh
for (display, video_queue) in self.video_queue_map.read().unwrap().iter() {
let tolerable = std::cmp::min(decode_fps, video_queue.capacity() / 2);
if ctl.refresh_times < 10 // enough
&& (len > tolerable
&& (ctl.refresh_times == 0 || ctl.last_refresh_instant.elapsed().as_secs() > 10))
if ctl.refresh_times < 20 // enough
&& (len > tolerable
&& (ctl.refresh_times == 0 || ctl.last_refresh_instant.elapsed().as_secs() > 10))
{
// Refresh causes client set_display, left frames cause flickering.
while let Some(_) = video_queue.pop() {}
@ -1086,6 +1101,9 @@ impl<T: InvokeUiSession> Remote<T> {
}
self.video_sender.send(MediaData::VideoQueue(display)).ok();
}
self.fps_control
.last_active_time
.insert(display, Instant::now());
}
Some(message::Union::Hash(hash)) => {
self.handler
@ -1840,8 +1858,9 @@ struct FpsControl {
last_queue_size: usize,
refresh_times: usize,
last_refresh_instant: Instant,
last_auto_fps: Option<i32>,
last_auto_fps: Option<usize>,
idle_counter: usize,
last_active_time: HashMap<usize, Instant>,
}
impl Default for FpsControl {
@ -1850,8 +1869,9 @@ impl Default for FpsControl {
last_queue_size: Default::default(),
refresh_times: Default::default(),
last_refresh_instant: Instant::now(),
last_auto_fps: None,
last_auto_fps: Default::default(),
idle_counter: 0,
last_active_time: Default::default(),
}
}
}

4
src/ui_session_interface.rs
View File

@ -1692,7 +1692,7 @@ pub async fn io_loop<T: InvokeUiSession>(handler: Session<T>, round: u32) {
let frame_count_map: Arc<RwLock<HashMap<usize, usize>>> = Default::default();
let frame_count_map_cl = frame_count_map.clone();
let ui_handler = handler.ui_handler.clone();
let (video_sender, audio_sender, video_queue_map, decode_fps_map, chroma) =
let (video_sender, audio_sender, video_queue_map, decode_fps, chroma) =
start_video_audio_threads(
handler.clone(),
move |display: usize,
@ -1720,7 +1720,7 @@ pub async fn io_loop<T: InvokeUiSession>(handler: Session<T>, round: u32) {
receiver,
sender,
frame_count_map,
decode_fps_map,
decode_fps,
chroma,
);
remote.io_loop(&key, &token, round).await;