Merge pull request #3208 from fufesou/feat/macos_translate_mode

feat/macos translate mode

Cargo.lock

@@ -4554,12 +4554,13 @@ dependencies = [
 [[package]]
 name = "rdev"
 version = "0.5.0-2"
-source = "git+https://github.com/fufesou/rdev#cedc4e62744566775026af4b434ef799804c1130"
+source = "git+https://github.com/fufesou/rdev#5b9fb5e42117f44e0ce0fe7cf2bddf270c75f1dc"
 dependencies = [
  "cocoa",
  "core-foundation 0.9.3",
  "core-foundation-sys 0.8.3",
  "core-graphics 0.22.3",
+ "dispatch",
  "enum-map",
  "epoll",
  "inotify",

flutter/lib/desktop/widgets/remote_menubar.dart

@@ -1510,13 +1510,16 @@ class _RemoteMenubarState extends State<RemoteMenubar> {
             if (bind.sessionIsKeyboardModeSupported(
                 id: widget.id, mode: mode.key)) {
               if (mode.key == 'translate') {
-                if (!Platform.isWindows ||
+                if (Platform.isLinux ||
-                    widget.ffi.ffiModel.pi.platform != kPeerPlatformWindows) {
+                    widget.ffi.ffiModel.pi.platform == kPeerPlatformLinux) {
                   continue;
                 }
               }
-              list.add(MenuEntryRadioOption(
+              var text = translate(mode.menu);
-                  text: translate(mode.menu), value: mode.key));
+              if (mode.key == 'translate') {
+                text = '$text beta';
+              }
+              list.add(MenuEntryRadioOption(text: text, value: mode.key));
             }
           }
           return list;

libs/enigo/src/win/win_impl.rs

@@ -39,7 +39,7 @@ fn mouse_event(flags: u32, data: u32, dx: i32, dy: i32) -> DWORD {
     unsafe { SendInput(1, &mut input as LPINPUT, size_of::<INPUT>() as c_int) }
 }
 
-fn keybd_event(flags: u32, vk: u16, scan: u16) -> DWORD {
+fn keybd_event(mut flags: u32, vk: u16, scan: u16) -> DWORD {
     let mut scan = scan;
     unsafe {
         // https://github.com/rustdesk/rustdesk/issues/366
@@ -52,35 +52,33 @@ fn keybd_event(flags: u32, vk: u16, scan: u16) -> DWORD {
             scan = MapVirtualKeyExW(vk as _, 0, LAYOUT) as _;
         }
     }
-    let mut input: INPUT = unsafe { std::mem::MaybeUninit::zeroed().assume_init() };
-    input.type_ = INPUT_KEYBOARD;
-    unsafe {
-        let dst_ptr = (&mut input.u as *mut _) as *mut u8;
-        let flags = match vk as _ {
-            winapi::um::winuser::VK_HOME |
-            winapi::um::winuser::VK_UP |
-            winapi::um::winuser::VK_PRIOR |
-            winapi::um::winuser::VK_LEFT |
-            winapi::um::winuser::VK_RIGHT |
-            winapi::um::winuser::VK_END |
-            winapi::um::winuser::VK_DOWN |
-            winapi::um::winuser::VK_NEXT |
-            winapi::um::winuser::VK_INSERT | 
-            winapi::um::winuser::VK_DELETE => flags | winapi::um::winuser::KEYEVENTF_EXTENDEDKEY,
-            _ => flags,
-        };
 
-        let k = KEYBDINPUT {
+    if flags & KEYEVENTF_UNICODE == 0 {
+        if scan >> 8 == 0xE0 || scan >> 8 == 0xE1 {
+            flags |= winapi::um::winuser::KEYEVENTF_EXTENDEDKEY;
+        }
+    }
+    let mut union: INPUT_u = unsafe { std::mem::zeroed() };
+    unsafe {
+        *union.ki_mut() = KEYBDINPUT {
             wVk: vk,
             wScan: scan,
             dwFlags: flags,
             time: 0,
             dwExtraInfo: ENIGO_INPUT_EXTRA_VALUE,
         };
-        let src_ptr = (&k as *const _) as *const u8;
-        std::ptr::copy_nonoverlapping(src_ptr, dst_ptr, size_of::<KEYBDINPUT>());
     }
-    unsafe { SendInput(1, &mut input as LPINPUT, size_of::<INPUT>() as c_int) }
+    let mut inputs = [INPUT {
+        type_: INPUT_KEYBOARD,
+        u: union,
+    }; 1];
+    unsafe {
+        SendInput(
+            inputs.len() as UINT,
+            inputs.as_mut_ptr(),
+            size_of::<INPUT>() as c_int,
+        )
+    }
 }
 
 fn get_error() -> String {

libs/hbb_common/protos/message.proto

@@ -201,6 +201,8 @@ message KeyEvent {
   bool press = 2;
   oneof union {
     ControlKey control_key = 3;
+    // high word, sym key code. win: virtual-key code, linux: keysym ?, macos: 
+    // low word, position key code. win: scancode, linux: key code, macos: key code
     uint32 chr = 4;
     uint32 unicode = 5;
     string seq = 6;

src/flutter_ffi.rs

@@ -20,7 +20,6 @@ use std::{
     os::raw::c_char,
     str::FromStr,
 };
-use crate::ui_session_interface::InvokeUiSession;
 
 // use crate::hbbs_http::account::AuthResult;
 

src/keyboard.rs

@@ -18,6 +18,13 @@ use std::{
 #[cfg(windows)]
 static mut IS_ALT_GR: bool = false;
 
+#[allow(dead_code)]
+const OS_LOWER_WINDOWS: &str = "windows";
+#[allow(dead_code)]
+const OS_LOWER_LINUX: &str = "linux";
+#[allow(dead_code)]
+const OS_LOWER_MACOS: &str = "macos";
+
 #[cfg(any(target_os = "windows", target_os = "macos"))]
 static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false);
 
@@ -202,6 +209,9 @@ pub fn update_grab_get_key_name() {
 #[cfg(target_os = "windows")]
 static mut IS_0X021D_DOWN: bool = false;
 
+#[cfg(target_os = "macos")]
+static mut IS_LEFT_OPTION_DOWN: bool = false;
+
 pub fn start_grab_loop() {
     #[cfg(any(target_os = "windows", target_os = "macos"))]
     std::thread::spawn(move || {
@@ -213,6 +223,7 @@ pub fn start_grab_loop() {
 
             let mut _keyboard_mode = KeyboardMode::Map;
             let _scan_code = event.scan_code;
+            let _code = event.code;
             let res = if KEYBOARD_HOOKED.load(Ordering::SeqCst) {
                 _keyboard_mode = client::process_event(&event, None);
                 if is_press {
@@ -246,6 +257,13 @@ pub fn start_grab_loop() {
                 }
             }
 
+            #[cfg(target_os = "macos")]
+            unsafe {
+                if _code as u32 == rdev::kVK_Option {
+                    IS_LEFT_OPTION_DOWN = is_press;
+                }
+            }
+
             return res;
         };
         let func = move |event: Event| match event.event_type {
@@ -253,11 +271,13 @@ pub fn start_grab_loop() {
             EventType::KeyRelease(key) => try_handle_keyboard(event, key, false),
             _ => Some(event),
         };
+        #[cfg(target_os = "macos")]
+        rdev::set_is_main_thread(false);
+        #[cfg(target_os = "windows")]
+        rdev::set_event_popup(false);
         if let Err(error) = rdev::grab(func) {
             log::error!("rdev Error: {:?}", error)
         }
-        #[cfg(target_os = "windows")]
-        rdev::set_event_popup(false);
     });
 
     #[cfg(target_os = "linux")]
@@ -395,13 +415,16 @@ pub fn event_to_key_events(
         _ => {}
     }
 
+    let mut peer = get_peer_platform().to_lowercase();
+    peer.retain(|c| !c.is_whitespace());
+
     key_event.mode = keyboard_mode.into();
     let mut key_events = match keyboard_mode {
-        KeyboardMode::Map => match map_keyboard_mode(event, key_event) {
+        KeyboardMode::Map => match map_keyboard_mode(peer.as_str(), event, key_event) {
             Some(event) => [event].to_vec(),
             None => Vec::new(),
         },
-        KeyboardMode::Translate => translate_keyboard_mode(event, key_event),
+        KeyboardMode::Translate => translate_keyboard_mode(peer.as_str(), event, key_event),
         _ => {
             #[cfg(not(any(target_os = "android", target_os = "ios")))]
             {
@@ -424,7 +447,6 @@ pub fn event_to_key_events(
             }
         }
     }
-
     key_events
 }
 
@@ -698,7 +720,7 @@ pub fn legacy_keyboard_mode(event: &Event, mut key_event: KeyEvent) -> Vec<KeyEv
     events
 }
 
-pub fn map_keyboard_mode(event: &Event, mut key_event: KeyEvent) -> Option<KeyEvent> {
+pub fn map_keyboard_mode(peer: &str, event: &Event, mut key_event: KeyEvent) -> Option<KeyEvent> {
     match event.event_type {
         EventType::KeyPress(..) => {
             key_event.down = true;
@@ -709,12 +731,9 @@ pub fn map_keyboard_mode(event: &Event, mut key_event: KeyEvent) -> Option<KeyEv
         _ => return None,
     };
 
-    let mut peer = get_peer_platform().to_lowercase();
-    peer.retain(|c| !c.is_whitespace());
-
     #[cfg(target_os = "windows")]
-    let keycode = match peer.as_str() {
+    let keycode = match peer {
-        "windows" => {
+        OS_LOWER_WINDOWS => {
             // https://github.com/rustdesk/rustdesk/issues/1371
             // Filter scancodes that are greater than 255 and the hight word is not 0xE0.
             if event.scan_code > 255 && (event.scan_code >> 8) != 0xE0 {
@@ -722,7 +741,7 @@ pub fn map_keyboard_mode(event: &Event, mut key_event: KeyEvent) -> Option<KeyEv
             }
             event.scan_code
         }
-        "macos" => {
+        OS_LOWER_MACOS => {
             if hbb_common::config::LocalConfig::get_kb_layout_type() == "ISO" {
                 rdev::win_scancode_to_macos_iso_code(event.scan_code)?
             } else {
@@ -732,15 +751,15 @@ pub fn map_keyboard_mode(event: &Event, mut key_event: KeyEvent) -> Option<KeyEv
         _ => rdev::win_scancode_to_linux_code(event.scan_code)?,
     };
     #[cfg(target_os = "macos")]
-    let keycode = match peer.as_str() {
+    let keycode = match peer {
-        "windows" => rdev::macos_code_to_win_scancode(event.code as _)?,
+        OS_LOWER_WINDOWS => rdev::macos_code_to_win_scancode(event.code as _)?,
-        "macos" => event.code as _,
+        OS_LOWER_MACOS => event.code as _,
         _ => rdev::macos_code_to_linux_code(event.code as _)?,
     };
     #[cfg(target_os = "linux")]
-    let keycode = match peer.as_str() {
+    let keycode = match peer {
-        "windows" => rdev::linux_code_to_win_scancode(event.code as _)?,
+        OS_LOWER_WINDOWS => rdev::linux_code_to_win_scancode(event.code as _)?,
-        "macos" => {
+        OS_LOWER_MACOS => {
             if hbb_common::config::LocalConfig::get_kb_layout_type() == "ISO" {
                 rdev::linux_code_to_macos_iso_code(event.code as _)?
             } else {
@@ -759,12 +778,14 @@ pub fn map_keyboard_mode(event: &Event, mut key_event: KeyEvent) -> Option<KeyEv
 fn try_fill_unicode(event: &Event, key_event: &KeyEvent, events: &mut Vec<KeyEvent>) {
     match &event.unicode {
         Some(unicode_info) => {
-            for code in &unicode_info.unicode {
+            if let Some(name) = &unicode_info.name {
+                if name.len() > 0 {
                     let mut evt = key_event.clone();
-                evt.set_unicode(*code as _);
+                    evt.set_seq(name.to_string());
                     events.push(evt);
                 }
             }
+        }
         None => {}
     }
 }
@@ -783,43 +804,40 @@ fn is_hot_key_modifiers_down() -> bool {
     return false;
 }
 
-pub fn translate_virtual_keycode(event: &Event, mut key_event: KeyEvent) -> Option<KeyEvent> {
+#[inline]
-    match event.event_type {
+#[cfg(target_os = "windows")]
-        EventType::KeyPress(..) => {
+pub fn translate_key_code(peer: &str, event: &Event, key_event: KeyEvent) -> Option<KeyEvent> {
-            key_event.down = true;
+    let mut key_event = map_keyboard_mode(peer, event, key_event)?;
-        }
+    key_event.set_chr((key_event.chr() & 0x0000FFFF) | ((event.code as u32) << 16));
-        EventType::KeyRelease(..) => {
-            key_event.down = false;
-        }
-        _ => return None,
-    };
-
-    let mut peer = get_peer_platform().to_lowercase();
-    peer.retain(|c| !c.is_whitespace());
-
-    // #[cfg(target_os = "windows")]
-    // let keycode = match peer.as_str() {
-    //     "windows" => event.code,
-    //     "macos" => {
-    //         if hbb_common::config::LocalConfig::get_kb_layout_type() == "ISO" {
-    //             rdev::win_scancode_to_macos_iso_code(event.scan_code)?
-    //         } else {
-    //             rdev::win_scancode_to_macos_code(event.scan_code)?
-    //         }
-    //     }
-    //     _ => rdev::win_scancode_to_linux_code(event.scan_code)?,
-    // };
-
-    key_event.set_chr(event.code as _);
     Some(key_event)
 }
 
-pub fn translate_keyboard_mode(event: &Event, key_event: KeyEvent) -> Vec<KeyEvent> {
+#[inline]
+#[cfg(not(target_os = "windows"))]
+pub fn translate_key_code(peer: &str, event: &Event, key_event: KeyEvent) -> Option<KeyEvent> {
+    map_keyboard_mode(peer, event, key_event)
+}
+
+pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) -> Vec<KeyEvent> {
     let mut events: Vec<KeyEvent> = Vec::new();
     if let Some(unicode_info) = &event.unicode {
         if unicode_info.is_dead {
+            #[cfg(target_os = "macos")]
+            if peer != OS_LOWER_MACOS && unsafe { IS_LEFT_OPTION_DOWN } {
+                // try clear dead key state
+                // rdev::clear_dead_key_state();
+            } else {
                 return events;
             }
+            #[cfg(not(target_os = "macos"))]
+            return events;
+        }
+    }
+
+    #[cfg(target_os = "macos")]
+    // ignore right option key
+    if event.code as u32 == rdev::kVK_RightOption {
+        return events;
     }
 
     #[cfg(target_os = "windows")]
@@ -847,11 +865,16 @@ pub fn translate_keyboard_mode(event: &Event, key_event: KeyEvent) -> Vec<KeyEve
         }
     }
 
-    #[cfg(not(target_os = "windows"))]
+    #[cfg(target_os = "linux")]
     try_fill_unicode(event, &key_event, &mut events);
 
+    #[cfg(target_os = "macos")]
+    if !unsafe { IS_LEFT_OPTION_DOWN } {
+        try_fill_unicode(event, &key_event, &mut events);
+    }
+
     if events.is_empty() {
-        if let Some(evt) = translate_virtual_keycode(event, key_event) {
+        if let Some(evt) = translate_key_code(peer, event, key_event) {
             events.push(evt);
         }
     }

src/server/input_service.rs

@@ -1082,21 +1082,28 @@ fn legacy_keyboard_mode(evt: &KeyEvent) {
 }
 
 #[cfg(target_os = "windows")]
-fn translate_process_virtual_keycode(vk: u32, down: bool) {
+fn translate_process_code(code: u32, down: bool) {
     crate::platform::windows::try_change_desktop();
-    sim_rdev_rawkey_virtual(vk, down);
+    match code >> 16 {
+        0 => sim_rdev_rawkey_position(code, down),
+        vk_code => sim_rdev_rawkey_virtual(vk_code, down),
+    };
 }
 
 fn translate_keyboard_mode(evt: &KeyEvent) {
-    match evt.union {
+    match &evt.union {
-        Some(key_event::Union::Unicode(_unicode)) => {
+        Some(key_event::Union::Seq(seq)) => {
-            #[cfg(target_os = "windows")]
+            ENIGO.lock().unwrap().key_sequence(seq);
-            allow_err!(rdev::simulate_unicode(_unicode as _));
         }
         Some(key_event::Union::Chr(..)) =>
         {
             #[cfg(target_os = "windows")]
-            translate_process_virtual_keycode(evt.chr(), evt.down)
+            translate_process_code(evt.chr(), evt.down);
+            #[cfg(not(target_os = "windows"))]
+            sim_rdev_rawkey_position(evt.chr(), evt.down);
+        }
+        Some(key_event::Union::Unicode(..)) => {
+            // Do not handle unicode for now.
         }
         _ => {
             log::debug!("Unreachable. Unexpected key event {:?}", &evt);