fix kill occupied ipc process, find with enumerate, kill with NtTerminateProcess (#8289)

* I reproduced the issue, that process did't have title, couldn't be connected
  to and taskkill not work
* Test whether ipc is opccupied with enumerating named pipe
* With NtTerminateProcess, it was killed successfully.
* There is a way to find the exact process which occupy the ipc, I have
  not check it, it's from https://github.com/winsiderss/systeminformer

Signed-off-by: 21pages <sunboeasy@gmail.com>

src/common.rs

@@ -1700,6 +1700,15 @@ pub fn read_custom_client(config: &str) {
     }
 }
 
+#[inline]
+pub fn is_empty_uni_link(arg: &str) -> bool {
+    let prefix = crate::get_uri_prefix();
+    if !arg.starts_with(&prefix) {
+        return false;
+    }
+    arg[prefix.len()..].chars().all(|c| c == '/')
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

src/core_main.rs

@@ -22,15 +22,6 @@ macro_rules! my_println{
     };
 }
 
-#[inline]
-fn is_empty_uni_link(arg: &str) -> bool {
-    let prefix = crate::get_uri_prefix();
-    if !arg.starts_with(&prefix) {
-        return false;
-    }
-    arg[prefix.len()..].chars().all(|c| c == '/')
-}
-
 /// shared by flutter and sciter main function
 ///
 /// [Note]
@@ -168,7 +159,7 @@ pub fn core_main() -> Option<Vec<String>> {
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     init_plugins(&args);
     log::info!("main start args:{:?}", args);
-    if args.is_empty() || is_empty_uni_link(&args[0]) {
+    if args.is_empty() || crate::common::is_empty_uni_link(&args[0]) {
         std::thread::spawn(move || crate::start_server(false));
     } else {
         #[cfg(windows)]

src/ipc.rs

@@ -970,10 +970,27 @@ pub async fn test_rendezvous_server() -> ResultType<()> {
     Ok(())
 }
 
-#[tokio::main(flavor = "current_thread")]
+#[cfg(windows)]
-pub async fn test_ipc_connection() -> ResultType<()> {
+pub fn is_ipc_file_exist(suffix: &str) -> ResultType<bool> {
-    connect(1000, "").await?;
+    let file_name = format!("{}\\query{}", crate::get_app_name(), suffix);
-    Ok(())
+    let mut err = None;
+    for entry in std::fs::read_dir("\\\\.\\pipe\\")? {
+        match entry {
+            Ok(entry) => {
+                if entry.file_name().into_string().unwrap_or_default() == file_name {
+                    return Ok(true);
+                }
+            }
+            Err(e) => {
+                err = Some(e);
+            }
+        }
+    }
+    if let Some(e) = err {
+        Err(e.into())
+    } else {
+        Ok(false)
+    }
 }
 
 #[tokio::main(flavor = "current_thread")]

src/platform/windows.rs

@@ -18,7 +18,7 @@ use hbb_common::{
 use std::process::{Command, Stdio};
 use std::{
     collections::HashMap,
-    ffi::OsString,
+    ffi::{CString, OsString},
     fs, io,
     io::prelude::*,
     mem,
@@ -36,6 +36,7 @@ use winapi::{
     um::{
         errhandlingapi::GetLastError,
         handleapi::CloseHandle,
+        libloaderapi::{GetProcAddress, LoadLibraryA},
         minwinbase::STILL_ACTIVE,
         processthreadsapi::{
             GetCurrentProcess, GetCurrentProcessId, GetExitCodeProcess, OpenProcess,
@@ -47,8 +48,8 @@ use winapi::{
         wingdi::*,
         winnt::{
             TokenElevation, ES_AWAYMODE_REQUIRED, ES_CONTINUOUS, ES_DISPLAY_REQUIRED,
-            ES_SYSTEM_REQUIRED, HANDLE, PROCESS_QUERY_LIMITED_INFORMATION, TOKEN_ELEVATION,
+            ES_SYSTEM_REQUIRED, HANDLE, PROCESS_ALL_ACCESS, PROCESS_QUERY_LIMITED_INFORMATION,
-            TOKEN_QUERY,
+            TOKEN_ELEVATION, TOKEN_QUERY,
         },
         winreg::HKEY_CURRENT_USER,
         winuser::*,
@@ -2420,36 +2421,80 @@ pub fn is_x64() -> bool {
     unsafe { sys_info.u.s().wProcessorArchitecture == PROCESSOR_ARCHITECTURE_AMD64 }
 }
 
-#[cfg(feature = "flutter")]
+pub fn try_kill_rustdesk_main_window_process() -> ResultType<()> {
-pub fn try_kill_flutter_main_window_process() {
+    // Kill rustdesk.exe without extra arg, should only be called by --server
-    // It's called when --server failed to start ipc, because the ipc may be occupied by the main window process.
+    // We can find the exact process which occupies the ipc, see more from https://github.com/winsiderss/systeminformer
-    // When --service quit the ipc process, ipc process will call std::process::exit, std::process::exit not work may be the reason.
+    log::info!("try kill rustdesk main window process");
-    // FindWindow not work in --service, https://forums.codeguru.com/showthread.php?169091-FindWindow-in-service
+    use hbb_common::sysinfo::System;
-    log::info!("try kill flutter main window process");
+    let mut sys = System::new();
+    sys.refresh_processes();
+    let my_uid = sys
+        .process((std::process::id() as usize).into())
+        .map(|x| x.user_id())
+        .unwrap_or_default();
+    let my_pid = std::process::id();
+    let app_name = crate::get_app_name().to_lowercase();
+    if app_name.is_empty() {
+        bail!("app name is empty");
+    }
+    for (_, p) in sys.processes().iter() {
+        let p_name = p.name().to_lowercase();
+        // name equal
+        if !(p_name == app_name || p_name == app_name.clone() + ".exe") {
+            continue;
+        }
+        // arg more than 1
+        if p.cmd().len() < 1 {
+            continue;
+        }
+        // first arg contain app name
+        if !p.cmd()[0].to_lowercase().contains(&p_name) {
+            continue;
+        }
+        // only one arg or the second arg is empty uni link
+        let is_empty_uni = p.cmd().len() == 2 && crate::common::is_empty_uni_link(&p.cmd()[1]);
+        if !(p.cmd().len() == 1 || is_empty_uni) {
+            continue;
+        }
+        // skip self
+        if p.pid().as_u32() == my_pid {
+            continue;
+        }
+        // because we call it with --server, so we can check user_id, remove this if call it with user process
+        if p.user_id() == my_uid {
+            log::info!("user id equal, continue");
+            continue;
+        }
+        log::info!("try kill process: {:?}, pid = {:?}", p.cmd(), p.pid());
+        nt_terminate_process(p.pid().as_u32())?;
+        log::info!("kill process success: {:?}, pid = {:?}", p.cmd(), p.pid());
+        return Ok(());
+    }
+    bail!("failed to find rustdesk main window process");
+}
+
+fn nt_terminate_process(process_id: DWORD) -> ResultType<()> {
+    type NtTerminateProcess = unsafe extern "system" fn(HANDLE, DWORD) -> DWORD;
     unsafe {
-        let window_name = wide_string(&crate::get_app_name());
+        let h_module = LoadLibraryA(CString::new("ntdll.dll")?.as_ptr());
-        let class_name = wide_string(FLUTTER_RUNNER_WIN32_WINDOW_CLASS);
+        if !h_module.is_null() {
-        let hwnd = FindWindowW(class_name.as_ptr(), window_name.as_ptr());
+            let f_nt_terminate_process: NtTerminateProcess = std::mem::transmute(GetProcAddress(
-        if hwnd.is_null() {
+                h_module,
-            log::info!("not found flutter main window");
+                CString::new("NtTerminateProcess")?.as_ptr(),
-            return;
+            ));
-        }
+            let h_token = OpenProcess(PROCESS_ALL_ACCESS, 0, process_id);
-        let mut process_id: u32 = 0;
+            if !h_token.is_null() {
-        GetWindowThreadProcessId(hwnd, &mut process_id as *mut u32);
+                if f_nt_terminate_process(h_token, 1) == 0 {
-        if process_id == 0 {
+                    log::info!("terminate process {} success", process_id);
-            log::info!("failed to get flutter window process id");
+                    return Ok(());
-            return;
+                } else {
-        }
+                    bail!("NtTerminateProcess {} failed", process_id);
-        let output = Command::new("taskkill")
+                }
-            .arg("/F")
+            } else {
-            .arg("/PID")
+                bail!("OpenProcess {} failed", process_id);
-            .arg(process_id.to_string())
+            }
-            .output()
-            .expect("Failed to execute command");
-        if output.status.success() {
-            log::info!("kill flutter main window process success");
         } else {
-            log::error!("kill flutter main window process failed");
+            bail!("Failed to load ntdll.dll");
         }
     }
 }

src/server.rs

@@ -470,10 +470,12 @@ pub async fn start_server(is_server: bool) {
         std::thread::spawn(move || {
             if let Err(err) = crate::ipc::start("") {
                 log::error!("Failed to start ipc: {}", err);
-                #[cfg(all(windows, feature = "flutter"))]
+                #[cfg(windows)]
-                if crate::is_server() && crate::ipc::test_ipc_connection().is_ok() {
+                if crate::is_server() && crate::ipc::is_ipc_file_exist("").unwrap_or(false) {
                     log::error!("ipc is occupied by another process, try kill it");
-                    crate::platform::try_kill_flutter_main_window_process();
+                    if let Err(e) = crate::platform::try_kill_rustdesk_main_window_process() {
+                        log::error!("kill failed: {}", e);
+                    }
                 }
                 std::process::exit(-1);
             }