feat: map mode and translate mode receive side

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -307,7 +307,7 @@ class InputService : AccessibilityService() {
 
                 } else {
                     KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
-                        Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd")
+                        Log.d(logTag, "event $event text $text start $textSelectionStart end $textSelectionEnd")
                         if (isShowingHint) {
                             this.fakeEditTextForTextStateCalculation?.setText(null)
                         } else {

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt

@@ -1,6 +1,7 @@
 package hbb;
 import android.view.KeyEvent
 import android.view.KeyCharacterMap
+import hbb.MessageOuterClass.KeyboardMode
 import hbb.MessageOuterClass.ControlKey
 
 object KeyEventConverter {
@@ -10,12 +11,14 @@ object KeyEventConverter {
 
         android.util.Log.d(tag, "proto: $keyEventProto")
 
-        if (keyEventProto.hasUnicode()) {
+        val keyboardMode = keyEventProto.getMode()
-            chrValue = 
-        }
 
         if (keyEventProto.hasChr()) {
-            chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+            if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) {
+                chrValue = keyEventProto.getChr()
+            } else {
+                chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+            }
         } else if (keyEventProto.hasControlKey()) {
             chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
         }
@@ -29,6 +32,8 @@ object KeyEventConverter {
             }
         }
 
+        android.util.Log.d(tag, "modifiers: $modifiersList")
+
         var action = 0
         if (keyEventProto.getDown()) {
             action = KeyEvent.ACTION_DOWN

src/keyboard.rs

@@ -163,6 +163,21 @@ pub mod client {
         }
     }
 
+    pub fn map_key_to_control_key(key: &rdev::Key) -> Option<ControlKey> {
+        match key {
+            Key::Alt => Some(ControlKey::Alt),
+            Key::ShiftLeft => Some(ControlKey::Shift),
+            Key::ControlLeft => Some(ControlKey::Control),
+            Key::MetaLeft => Some(ControlKey::Meta),
+            Key::AltGr => Some(ControlKey::RAlt),
+            Key::ShiftRight => Some(ControlKey::RShift),
+            Key::ControlRight => Some(ControlKey::RControl),
+            Key::MetaRight => Some(ControlKey::RWin),
+            _ => None,
+        }
+
+    }
+
     pub fn event_lock_screen() -> KeyEvent {
         let mut key_event = KeyEvent::new();
         key_event.set_control_key(ControlKey::LockScreen);
@@ -355,7 +370,7 @@ pub fn get_keyboard_mode_enum(keyboard_mode: &str) -> KeyboardMode {
 }
 
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 pub fn is_modifier(key: &rdev::Key) -> bool {
     matches!(
         key,
@@ -1050,7 +1065,7 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) -
 pub fn keycode_to_rdev_key(keycode: u32) -> Key {
     #[cfg(target_os = "windows")]
     return rdev::win_key_from_scancode(keycode);
-    #[cfg(any(target_os = "linux", target_os = "android"))]
+    #[cfg(any(target_os = "linux"))]
     return rdev::linux_key_from_code(keycode);
     #[cfg(target_os = "macos")]
     return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default());

src/server/connection.rs

@@ -38,10 +38,12 @@ use hbb_common::{
         sync::mpsc,
         time::{self, Duration, Instant, Interval},
     },
-    tokio_util::codec::{BytesCodec, Framed},
+    tokio_util::codec::{BytesCodec, Framed}, protobuf::EnumOrUnknown,
 };
 #[cfg(any(target_os = "android", target_os = "ios"))]
 use scrap::android::{call_main_service_pointer_input, call_main_service_key_event};
+#[cfg(target_os = "android")]
+use crate::keyboard::client::map_key_to_control_key;
 use serde_json::{json, value::Value};
 use sha2::{Digest, Sha256};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -53,7 +55,7 @@ use std::{
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use system_shutdown;
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 use std::collections::HashSet;
 
 pub type Sender = mpsc::UnboundedSender<(Instant, Arc<Message>)>;
@@ -211,7 +213,7 @@ pub struct Connection {
     voice_call_request_timestamp: Option<NonZeroI64>,
     audio_input_device_before_voice_call: Option<String>,
     options_in_login: Option<OptionMessage>,
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    #[cfg(not(any(target_os = "ios")))]
     pressed_modifiers: HashSet<rdev::Key>,
     #[cfg(all(target_os = "linux", feature = "linux_headless"))]
     #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -348,7 +350,7 @@ impl Connection {
             voice_call_request_timestamp: None,
             audio_input_device_before_voice_call: None,
             options_in_login: None,
-            #[cfg(not(any(target_os = "android", target_os = "ios")))]
+            #[cfg(not(any(target_os = "ios")))]
             pressed_modifiers: Default::default(),
             #[cfg(all(target_os = "linux", feature = "linux_headless"))]
             #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -1726,6 +1728,12 @@ impl Connection {
                 Some(message::Union::KeyEvent(..)) => {}
                 #[cfg(any(target_os = "android"))]
                 Some(message::Union::KeyEvent(mut me)) => {
+                    let is_press = if cfg!(target_os = "linux") {
+                        (me.press || me.down) && !crate::is_modifier(&me)
+                    } else {
+                        me.press
+                    };
+
                     let key = match me.mode.enum_value() {
                         Ok(KeyboardMode::Map) => {
                             Some(crate::keyboard::keycode_to_rdev_key(me.chr()))
@@ -1738,7 +1746,28 @@ impl Connection {
                             }
                         }
                         _ => None,
-                    };
+                    }
+                    .filter(crate::keyboard::is_modifier);
+                    log::debug!("key:{:?}", key);
+
+                    if let Some(key) = key {
+                        if is_press {
+                            self.pressed_modifiers.insert(key);
+                        } else {
+                            self.pressed_modifiers.remove(&key);
+                        }
+                    }
+
+                    let mut modifiers = vec![];
+
+                    for key in self.pressed_modifiers.iter() {
+                        if let Some(control_key) = map_key_to_control_key(key) {
+                            modifiers.push(EnumOrUnknown::new(control_key));
+                        }
+                    }
+
+                    me.modifiers = modifiers;
+
                     let encode_result = me.write_to_bytes();
 
                     if let Ok(data) = encode_result {