diff --git a/src/server/input_service.rs b/src/server/input_service.rs
index 17f73fe5f..449f1da3e 100644
--- a/src/server/input_service.rs
+++ b/src/server/input_service.rs
@@ -319,6 +319,8 @@ fn handle_mouse_(evt: &MouseEvent, conn: i32) {
                         en.key_down(key.clone()).ok();
                         modifier_sleep();
                         to_release.push(key);
+                    } else {
+                        KEYS_DOWN.lock().unwrap().insert(ck.value(), Instant::now());
                     }
                 }
             }
@@ -534,6 +536,8 @@ fn handle_key_(evt: &KeyEvent) {
                             en.key_down(key.clone()).ok();
                             modifier_sleep();
                             to_release.push(key);
+                        } else {
+                            KEYS_DOWN.lock().unwrap().insert(ck.value(), Instant::now());
                         }
                     }
                 }