diff --git a/src/ipc.rs b/src/ipc.rs
index 34567b77a..b81da4f13 100644
--- a/src/ipc.rs
+++ b/src/ipc.rs
@@ -147,7 +147,7 @@ pub enum DataPortableService {
     Ping,
     Pong,
     ConnCount(Option<usize>),
-    Mouse(Vec<u8>),
+    Mouse((Vec<u8>, i32)),
     Key(Vec<u8>),
     RequestStart,
     WillClose,
diff --git a/src/server/connection.rs b/src/server/connection.rs
index 10ae92115..dd99270db 100644
--- a/src/server/connection.rs
+++ b/src/server/connection.rs
@@ -1088,7 +1088,7 @@ impl Connection {
                 }
                 let mut s = s.write().unwrap();
                 #[cfg(not(any(target_os = "android", target_os = "ios")))]
-                try_start_record_cursor_pos();
+                let _h = try_start_record_cursor_pos();
                 s.add_connection(self.inner.clone(), &noperms);
             }
         }
diff --git a/src/server/input_service.rs b/src/server/input_service.rs
index f49afb5bc..4d2997c78 100644
--- a/src/server/input_service.rs
+++ b/src/server/input_service.rs
@@ -341,13 +341,13 @@ const MOUSE_ACTIVE_DISTANCE: i32 = 5;
 
 static RECORD_CURSOR_POS_RUNNING: AtomicBool = AtomicBool::new(false);
 
-pub fn try_start_record_cursor_pos() {
+pub fn try_start_record_cursor_pos() -> Option<thread::JoinHandle<()>> {
     if RECORD_CURSOR_POS_RUNNING.load(Ordering::SeqCst) {
-        return;
+        return None;
     }
 
     RECORD_CURSOR_POS_RUNNING.store(true, Ordering::SeqCst);
-    thread::spawn(|| {
+    let handle = thread::spawn(|| {
         let interval = time::Duration::from_millis(33);
         loop {
             if !RECORD_CURSOR_POS_RUNNING.load(Ordering::SeqCst) {
@@ -365,6 +365,7 @@ pub fn try_start_record_cursor_pos() {
         }
         update_last_cursor_pos(INVALID_CURSOR_POS, INVALID_CURSOR_POS);
     });
+    Some(handle)
 }
 
 pub fn try_stop_record_cursor_pos() {
@@ -506,26 +507,7 @@ fn get_modifier_state(key: Key, en: &mut Enigo) -> bool {
     }
 }
 
-#[inline]
-fn update_latest_peer_input_cursor(evt: &MouseEvent, conn: i32, is_checked_movement: bool) {
-    if is_checked_movement || evt.mask & 0x7 == 0 {
-        let time = get_time();
-        *LATEST_PEER_INPUT_CURSOR.lock().unwrap() = Input {
-            time,
-            conn,
-            x: evt.x,
-            y: evt.y,
-        };
-    }
-}
-
 pub fn handle_mouse(evt: &MouseEvent, conn: i32) {
-    if !active_mouse_(conn) {
-        return;
-    }
-    #[cfg(windows)]
-    update_latest_peer_input_cursor(evt, conn, false);
-
     #[cfg(target_os = "macos")]
     if !is_server() {
         // having GUI, run main GUI thread, otherwise crash
@@ -534,7 +516,7 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) {
         return;
     }
     #[cfg(windows)]
-    crate::portable_service::client::handle_mouse(evt);
+    crate::portable_service::client::handle_mouse(evt, conn);
     #[cfg(not(windows))]
     handle_mouse_(evt, conn);
 }
@@ -750,8 +732,11 @@ fn active_mouse_(conn: i32) -> bool {
     }
 }
 
-// _conn is used by `update_latest_peer_input_cursor`, which is only enabled on "not Win".
-pub fn handle_mouse_(evt: &MouseEvent, _conn: i32) {
+pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
+    if !active_mouse_(conn) {
+        return;
+    }
+
     if EXITING.load(Ordering::SeqCst) {
         return;
     }
@@ -786,8 +771,12 @@ pub fn handle_mouse_(evt: &MouseEvent, _conn: i32) {
     match evt_type {
         0 => {
             en.mouse_move_to(evt.x, evt.y);
-            #[cfg(not(windows))]
-            update_latest_peer_input_cursor(evt, _conn, true);
+            *LATEST_PEER_INPUT_CURSOR.lock().unwrap() = Input {
+                conn,
+                time: get_time(),
+                x: evt.x,
+                y: evt.y,
+            };
         }
         1 => match buttons {
             0x01 => {
diff --git a/src/server/portable_service.rs b/src/server/portable_service.rs
index f0fb0799d..49648769d 100644
--- a/src/server/portable_service.rs
+++ b/src/server/portable_service.rs
@@ -245,10 +245,19 @@ pub mod server {
         threads.push(std::thread::spawn(|| {
             run_exit_check();
         }));
+        let record_pos_handle = crate::input_service::try_start_record_cursor_pos();
         for th in threads.drain(..) {
             th.join().unwrap();
             log::info!("thread joined");
         }
+
+        crate::input_service::try_stop_record_cursor_pos();
+        if let Some(handle) = record_pos_handle {
+            match handle.join() {
+                Ok(_) => log::info!("record_pos_handle joined"),
+                Err(e) => log::error!("record_pos_handle join error {:?}", &e),
+            }
+        }
     }
 
     fn run_exit_check() {
@@ -452,9 +461,9 @@ pub mod server {
                                             break;
                                         }
                                     }
-                                    Mouse(v) => {
+                                    Mouse((v, conn)) => {
                                         if let Ok(evt) = MouseEvent::parse_from_bytes(&v) {
-                                            crate::input_service::handle_mouse_(&evt, 0);
+                                            crate::input_service::handle_mouse_(&evt, conn);
                                         }
                                     }
                                     Key(v) => {
@@ -847,10 +856,12 @@ pub mod client {
         }
     }
 
-    fn handle_mouse_(evt: &MouseEvent) -> ResultType<()> {
+    fn handle_mouse_(evt: &MouseEvent, conn: i32) -> ResultType<()> {
         let mut v = vec![];
         evt.write_to_vec(&mut v)?;
-        ipc_send(Data::DataPortableService(DataPortableService::Mouse(v)))
+        ipc_send(Data::DataPortableService(DataPortableService::Mouse((
+            v, conn,
+        ))))
     }
 
     fn handle_key_(evt: &KeyEvent) -> ResultType<()> {
@@ -890,11 +901,11 @@ pub mod client {
         }
     }
 
-    pub fn handle_mouse(evt: &MouseEvent) {
+    pub fn handle_mouse(evt: &MouseEvent, conn: i32) {
         if RUNNING.lock().unwrap().clone() {
-            handle_mouse_(evt).ok();
+            handle_mouse_(evt, conn).ok();
         } else {
-            crate::input_service::handle_mouse_(evt, 0);
+            crate::input_service::handle_mouse_(evt, conn);
         }
     }