From b763ec30802222c64252c1a595a1cb754886c949 Mon Sep 17 00:00:00 2001 From: rustdesk Date: Thu, 23 Mar 2023 14:31:50 +0800 Subject: [PATCH] revert #560 for #3748 --- libs/hbb_common/protos/message.proto | 2 - libs/scrap/src/common/vpxcodec.rs | 3 +- src/client.rs | 58 ++++---------------- src/client/helper.rs | 80 ++-------------------------- src/client/io_loop.rs | 2 - src/server/audio_service.rs | 3 -- src/server/connection.rs | 18 +++---- src/server/video_service.rs | 1 - 8 files changed, 20 insertions(+), 147 deletions(-) diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index bb4492b74..6295c160b 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -25,7 +25,6 @@ message VideoFrame { EncodedVideoFrames h264s = 10; EncodedVideoFrames h265s = 11; } - int64 timestamp = 9; } message IdPk { @@ -513,7 +512,6 @@ message AudioFormat { message AudioFrame { bytes data = 1; - int64 timestamp = 2; } // Notify peer to show message box. diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index b91871c8f..3df9c0461 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -4,7 +4,7 @@ use hbb_common::anyhow::{anyhow, Context}; use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}; -use hbb_common::{get_time, ResultType}; +use hbb_common::ResultType; use crate::STRIDE_ALIGN; use crate::{codec::EncoderApi, ImageFormat}; @@ -287,7 +287,6 @@ impl VpxEncoder { frames: vp9s.into(), ..Default::default() }); - vf.timestamp = get_time(); msg_out.set_video_frame(vf); msg_out } diff --git a/src/client.rs b/src/client.rs index 0cd90e1f3..607b96635 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1,7 +1,7 @@ use std::{ collections::HashMap, net::SocketAddr, - ops::{Deref, Not}, + ops::Deref, str::FromStr, sync::{mpsc, Arc, Mutex, RwLock}, }; @@ -39,7 +39,6 @@ use hbb_common::{ tokio::time::Duration, AddrMangle, ResultType, Stream, }; -pub use helper::LatencyController; pub use helper::*; use scrap::{ codec::{Decoder, DecoderCfg}, @@ -707,19 +706,9 @@ pub struct AudioHandler { #[cfg(not(any(target_os = "android", target_os = "linux")))] audio_stream: Option>, channels: u16, - latency_controller: Arc>, - ignore_count: i32, } impl AudioHandler { - /// Create a new audio handler. - pub fn new(latency_controller: Arc>) -> Self { - AudioHandler { - latency_controller, - ..Default::default() - } - } - /// Start the audio playback. #[cfg(target_os = "linux")] fn start_audio(&mut self, format0: AudioFormat) -> ResultType<()> { @@ -802,24 +791,8 @@ impl AudioHandler { } /// Handle audio frame and play it. + #[inline] pub fn handle_frame(&mut self, frame: AudioFrame) { - if frame.timestamp != 0 { - if self - .latency_controller - .lock() - .unwrap() - .check_audio(frame.timestamp) - .not() - { - self.ignore_count += 1; - if self.ignore_count == 100 { - self.ignore_count = 0; - log::debug!("100 audio frames are ignored"); - } - return; - } - } - #[cfg(not(any(target_os = "android", target_os = "linux")))] if self.audio_stream.is_none() { return; @@ -914,7 +887,6 @@ impl AudioHandler { /// Video handler for the [`Client`]. pub struct VideoHandler { decoder: Decoder, - latency_controller: Arc>, pub rgb: Vec, recorder: Arc>>, record: bool, @@ -922,7 +894,7 @@ pub struct VideoHandler { impl VideoHandler { /// Create a new video handler. - pub fn new(latency_controller: Arc>) -> Self { + pub fn new() -> Self { VideoHandler { decoder: Decoder::new(DecoderCfg { vpx: VpxDecoderConfig { @@ -930,7 +902,6 @@ impl VideoHandler { num_threads: (num_cpus::get() / 2) as _, }, }), - latency_controller, rgb: Default::default(), recorder: Default::default(), record: false, @@ -938,14 +909,8 @@ impl VideoHandler { } /// Handle a new video frame. + #[inline] pub fn handle_frame(&mut self, vf: VideoFrame) -> ResultType { - if vf.timestamp != 0 { - // Update the latency controller with the latest timestamp. - self.latency_controller - .lock() - .unwrap() - .update_video(vf.timestamp); - } match &vf.union { Some(frame) => { let res = self.decoder.handle_video_frame( @@ -994,6 +959,7 @@ impl VideoHandler { } else { self.recorder = Default::default(); } + self.record = start; } } @@ -1696,11 +1662,8 @@ where let (video_sender, video_receiver) = mpsc::channel::(); let mut video_callback = video_callback; - let latency_controller = LatencyController::new(); - let latency_controller_cl = latency_controller.clone(); - std::thread::spawn(move || { - let mut video_handler = VideoHandler::new(latency_controller); + let mut video_handler = VideoHandler::new(); loop { if let Ok(data) = video_receiver.recv() { match data { @@ -1723,19 +1686,16 @@ where } log::info!("Video decoder loop exits"); }); - let audio_sender = start_audio_thread(Some(latency_controller_cl)); + let audio_sender = start_audio_thread(); return (video_sender, audio_sender); } /// Start an audio thread /// Return a audio [`MediaSender`] -pub fn start_audio_thread( - latency_controller: Option>>, -) -> MediaSender { - let latency_controller = latency_controller.unwrap_or(LatencyController::new()); +pub fn start_audio_thread() -> MediaSender { let (audio_sender, audio_receiver) = mpsc::channel::(); std::thread::spawn(move || { - let mut audio_handler = AudioHandler::new(latency_controller); + let mut audio_handler = AudioHandler::default(); loop { if let Ok(data) = audio_receiver.recv() { match data { diff --git a/src/client/helper.rs b/src/client/helper.rs index 20acd811a..a9696a8e8 100644 --- a/src/client/helper.rs +++ b/src/client/helper.rs @@ -1,82 +1,8 @@ -use std::{ - sync::{Arc, Mutex}, - time::Instant, -}; - use hbb_common::{ - log, - message_proto::{video_frame, VideoFrame, Message, VoiceCallRequest, VoiceCallResponse}, get_time, + get_time, + message_proto::{video_frame, Message, VideoFrame, VoiceCallRequest, VoiceCallResponse}, }; -const MAX_LATENCY: i64 = 500; -const MIN_LATENCY: i64 = 100; - -/// Latency controller for syncing audio with the video stream. -/// Only sync the audio to video, not the other way around. -#[derive(Debug)] -pub struct LatencyController { - last_video_remote_ts: i64, // generated on remote device - update_time: Instant, - allow_audio: bool, - audio_only: bool -} - -impl Default for LatencyController { - fn default() -> Self { - Self { - last_video_remote_ts: Default::default(), - update_time: Instant::now(), - allow_audio: Default::default(), - audio_only: false - } - } -} - -impl LatencyController { - /// Create a new latency controller. - pub fn new() -> Arc> { - Arc::new(Mutex::new(LatencyController::default())) - } - - /// Set whether this [LatencyController] should be working in audio only mode. - pub fn set_audio_only(&mut self, only: bool) { - self.audio_only = only; - } - - /// Update the latency controller with the latest video timestamp. - pub fn update_video(&mut self, timestamp: i64) { - self.last_video_remote_ts = timestamp; - self.update_time = Instant::now(); - } - - /// Check if the audio should be played based on the current latency. - pub fn check_audio(&mut self, timestamp: i64) -> bool { - // Compute audio latency. - let expected = self.update_time.elapsed().as_millis() as i64 + self.last_video_remote_ts; - let latency = if self.audio_only { - expected - } else { - expected - timestamp - }; - // Set MAX and MIN, avoid fixing too frequently. - if self.allow_audio { - if latency.abs() > MAX_LATENCY { - log::debug!("LATENCY > {}ms cut off, latency:{}", MAX_LATENCY, latency); - self.allow_audio = false; - } - } else { - if latency.abs() < MIN_LATENCY { - log::debug!("LATENCY < {}ms resume, latency:{}", MIN_LATENCY, latency); - self.allow_audio = true; - } - } - // No video frame here, which means the update time is not up to date. - // We manually update the time here. - self.update_time = Instant::now(); - self.allow_audio - } -} - #[derive(PartialEq, Debug, Clone)] pub enum CodecFormat { VP9, @@ -135,4 +61,4 @@ pub fn new_voice_call_response(request_timestamp: i64, accepted: bool) -> Messag let mut msg = Message::new(); msg.set_voice_call_response(resp); msg -} \ No newline at end of file +} diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index 989c7c95b..4a020d197 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -318,13 +318,11 @@ impl Remote { let mut msg = Message::new(); msg.set_audio_frame(frame.clone()); tx_audio.send(Data::Message(msg)).ok(); - log::debug!("send audio frame {}", frame.timestamp); } Some(message::Union::Misc(misc)) => { let mut msg = Message::new(); msg.set_misc(misc.clone()); tx_audio.send(Data::Message(msg)).ok(); - log::debug!("send audio misc {:?}", misc.audio_format()); } _ => {} }, diff --git a/src/server/audio_service.rs b/src/server/audio_service.rs index c7a720ded..bef1ac1f3 100644 --- a/src/server/audio_service.rs +++ b/src/server/audio_service.rs @@ -13,7 +13,6 @@ // https://github.com/krruzic/pulsectl use super::*; -use hbb_common::get_time; use magnum_opus::{Application::*, Channels::*, Encoder}; use std::sync::atomic::{AtomicBool, Ordering}; @@ -349,7 +348,6 @@ fn send_f32(data: &[f32], encoder: &mut Encoder, sp: &GenericService) { let mut msg_out = Message::new(); msg_out.set_audio_frame(AudioFrame { data: data.into(), - timestamp: get_time(), ..Default::default() }); sp.send(msg_out); @@ -369,7 +367,6 @@ fn send_f32(data: &[f32], encoder: &mut Encoder, sp: &GenericService) { let mut msg_out = Message::new(); msg_out.set_audio_frame(AudioFrame { data: data.into(), - timestamp: get_time(), ..Default::default() }); sp.send(msg_out); diff --git a/src/server/connection.rs b/src/server/connection.rs index 4b65013a3..1f89d905f 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -7,8 +7,7 @@ use crate::common::update_clipboard; use crate::portable_service::client as portable_client; use crate::{ client::{ - new_voice_call_request, new_voice_call_response, start_audio_thread, LatencyController, - MediaData, MediaSender, + new_voice_call_request, new_voice_call_response, start_audio_thread, MediaData, MediaSender, }, common::{get_default_sound_input, set_sound_input}, video_service, @@ -843,15 +842,16 @@ impl Connection { pi.hostname = DEVICE_NAME.lock().unwrap().clone(); pi.platform = "Android".into(); } - let mut platform_additions = serde_json::Map::new(); #[cfg(target_os = "linux")] { + let mut platform_additions = serde_json::Map::new(); if crate::platform::current_is_wayland() { platform_additions.insert("is_wayland".into(), json!(true)); } - } - if !platform_additions.is_empty() { - pi.platform_additions = serde_json::to_string(&platform_additions).unwrap_or("".into()); + if !platform_additions.is_empty() { + pi.platform_additions = + serde_json::to_string(&platform_additions).unwrap_or("".into()); + } } #[cfg(feature = "hwcodec")] @@ -1612,11 +1612,7 @@ impl Connection { if !self.disable_audio { // Drop the audio sender previously. drop(std::mem::replace(&mut self.audio_sender, None)); - // Start a audio thread to play the audio sent by peer. - let latency_controller = LatencyController::new(); - // No video frame will be sent here, so we need to disable latency controller, or audio check may fail. - latency_controller.lock().unwrap().set_audio_only(true); - self.audio_sender = Some(start_audio_thread(Some(latency_controller))); + self.audio_sender = Some(start_audio_thread()); allow_err!(self .audio_sender .as_ref() diff --git a/src/server/video_service.rs b/src/server/video_service.rs index affb5eb17..9d6eec85c 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -787,7 +787,6 @@ fn create_msg(vp9s: Vec) -> Message { frames: vp9s.into(), ..Default::default() }); - vf.timestamp = hbb_common::get_time(); msg_out.set_video_frame(vf); msg_out }