diff --git a/src/client.rs b/src/client.rs
index 03bbf5918..2922a488b 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -892,6 +892,8 @@ pub struct LoginConfigHandler {
     pub supported_encoding: Option<(bool, bool)>,
     pub restarting_remote_device: bool,
     pub force_relay: bool,
+    pub direct: Option<bool>,
+    pub received: bool,
 }
 
 impl Deref for LoginConfigHandler {
@@ -929,6 +931,8 @@ impl LoginConfigHandler {
         self.supported_encoding = None;
         self.restarting_remote_device = false;
         self.force_relay = !self.get_option("force-always-relay").is_empty();
+        self.direct = None;
+        self.received = false;
     }
 
     /// Check if the client should auto login.
@@ -1815,6 +1819,7 @@ pub trait Interface: Send + Clone + 'static + Sized {
     fn handle_login_error(&mut self, err: &str) -> bool;
     fn handle_peer_info(&mut self, pi: PeerInfo);
     fn set_force_relay(&mut self, direct: bool, received: bool);
+    fn set_connection_info(&mut self, direct: bool, received: bool);
     fn is_file_transfer(&self) -> bool;
     fn is_port_forward(&self) -> bool;
     fn is_rdp(&self) -> bool;
@@ -1990,11 +1995,10 @@ lazy_static::lazy_static! {
 /// * `title` - The title of the message.
 /// * `text` - The text of the message.
 #[inline]
-pub fn check_if_retry(msgtype: &str, title: &str, text: &str) -> bool {
+pub fn check_if_retry(msgtype: &str, title: &str, text: &str, retry_for_relay: bool) -> bool {
     msgtype == "error"
         && title == "Connection Error"
-        && (text.contains("10054")
-            || text.contains("104")
+        && ((text.contains("10054") || text.contains("104")) && retry_for_relay
             || (!text.to_lowercase().contains("offline")
                 && !text.to_lowercase().contains("exist")
                 && !text.to_lowercase().contains("handshake")
@@ -2002,7 +2006,8 @@ pub fn check_if_retry(msgtype: &str, title: &str, text: &str) -> bool {
                 && !text.to_lowercase().contains("resolve")
                 && !text.to_lowercase().contains("mismatch")
                 && !text.to_lowercase().contains("manually")
-                && !text.to_lowercase().contains("not allowed")))
+                && !text.to_lowercase().contains("not allowed")
+                && !text.to_lowercase().contains("reset by the peer")))
 }
 
 #[inline]
diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs
index 326857d3f..ceddbc004 100644
--- a/src/client/io_loop.rs
+++ b/src/client/io_loop.rs
@@ -107,6 +107,7 @@ impl<T: InvokeUiSession> Remote<T> {
                 SERVER_CLIPBOARD_ENABLED.store(true, Ordering::SeqCst);
                 SERVER_FILE_TRANSFER_ENABLED.store(true, Ordering::SeqCst);
                 self.handler.set_connection_type(peer.is_secured(), direct); // flutter -> connection_ready
+                self.handler.set_connection_info(direct, false);
 
                 // just build for now
                 #[cfg(not(windows))]
@@ -144,7 +145,10 @@ impl<T: InvokeUiSession> Remote<T> {
                                     }
                                     Ok(ref bytes) => {
                                         last_recv_time = Instant::now();
-                                        received = true;
+                                        if !received {
+                                            received = true;
+                                            self.handler.set_connection_info(direct, true);
+                                        }
                                         self.data_count.fetch_add(bytes.len(), Ordering::Relaxed);
                                         if !self.handle_msg_from_peer(bytes, &mut peer).await {
                                             break
diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs
index f6d5a0e94..9868b5bb1 100644
--- a/src/ui_session_interface.rs
+++ b/src/ui_session_interface.rs
@@ -657,7 +657,10 @@ impl<T: InvokeUiSession> Interface for Session<T> {
     }
 
     fn msgbox(&self, msgtype: &str, title: &str, text: &str, link: &str) {
-        let retry = check_if_retry(msgtype, title, text);
+        let direct = self.lc.read().unwrap().direct.unwrap_or_default();
+        let received = self.lc.read().unwrap().received;
+        let retry_for_relay = direct && !received;
+        let retry = check_if_retry(msgtype, title, text, retry_for_relay);
         self.ui_handler.msgbox(msgtype, title, text, link, retry);
     }
 
@@ -745,6 +748,12 @@ impl<T: InvokeUiSession> Interface for Session<T> {
         }
     }
 
+    fn set_connection_info(&mut self, direct: bool, received: bool) {
+        let mut lc = self.lc.write().unwrap();
+        lc.direct = Some(direct);
+        lc.received = received;
+    }
+
     fn set_force_relay(&mut self, direct: bool, received: bool) {
         let mut lc = self.lc.write().unwrap();
         lc.force_relay = false;