diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 50445589c..8e0582d17 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -307,7 +307,7 @@ class InputService : AccessibilityService() { } else { KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> - Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd") + Log.d(logTag, "event $event text $text start $textSelectionStart end $textSelectionEnd") if (isShowingHint) { this.fakeEditTextForTextStateCalculation?.setText(null) } else { diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt index f7273625e..47b237eaf 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -1,6 +1,7 @@ package hbb; import android.view.KeyEvent import android.view.KeyCharacterMap +import hbb.MessageOuterClass.KeyboardMode import hbb.MessageOuterClass.ControlKey object KeyEventConverter { @@ -10,12 +11,14 @@ object KeyEventConverter { android.util.Log.d(tag, "proto: $keyEventProto") - if (keyEventProto.hasUnicode()) { - chrValue = - } + val keyboardMode = keyEventProto.getMode() if (keyEventProto.hasChr()) { - chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) { + chrValue = keyEventProto.getChr() + } else { + chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + } } else if (keyEventProto.hasControlKey()) { chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey()) } @@ -29,6 +32,8 @@ object KeyEventConverter { } } + android.util.Log.d(tag, "modifiers: $modifiersList") + var action = 0 if (keyEventProto.getDown()) { action = KeyEvent.ACTION_DOWN diff --git a/src/keyboard.rs b/src/keyboard.rs index 46d0c8f9d..4a46d8934 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -163,6 +163,21 @@ pub mod client { } } + pub fn map_key_to_control_key(key: &rdev::Key) -> Option { + match key { + Key::Alt => Some(ControlKey::Alt), + Key::ShiftLeft => Some(ControlKey::Shift), + Key::ControlLeft => Some(ControlKey::Control), + Key::MetaLeft => Some(ControlKey::Meta), + Key::AltGr => Some(ControlKey::RAlt), + Key::ShiftRight => Some(ControlKey::RShift), + Key::ControlRight => Some(ControlKey::RControl), + Key::MetaRight => Some(ControlKey::RWin), + _ => None, + } + + } + pub fn event_lock_screen() -> KeyEvent { let mut key_event = KeyEvent::new(); key_event.set_control_key(ControlKey::LockScreen); @@ -355,7 +370,7 @@ pub fn get_keyboard_mode_enum(keyboard_mode: &str) -> KeyboardMode { } #[inline] -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] pub fn is_modifier(key: &rdev::Key) -> bool { matches!( key, @@ -1050,7 +1065,7 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) - pub fn keycode_to_rdev_key(keycode: u32) -> Key { #[cfg(target_os = "windows")] return rdev::win_key_from_scancode(keycode); - #[cfg(any(target_os = "linux", target_os = "android"))] + #[cfg(any(target_os = "linux"))] return rdev::linux_key_from_code(keycode); #[cfg(target_os = "macos")] return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default()); diff --git a/src/server/connection.rs b/src/server/connection.rs index 20e2c9e2e..7f22bd8f9 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -38,10 +38,12 @@ use hbb_common::{ sync::mpsc, time::{self, Duration, Instant, Interval}, }, - tokio_util::codec::{BytesCodec, Framed}, + tokio_util::codec::{BytesCodec, Framed}, protobuf::EnumOrUnknown, }; #[cfg(any(target_os = "android", target_os = "ios"))] use scrap::android::{call_main_service_pointer_input, call_main_service_key_event}; +#[cfg(target_os = "android")] +use crate::keyboard::client::map_key_to_control_key; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -53,7 +55,7 @@ use std::{ #[cfg(not(any(target_os = "android", target_os = "ios")))] use system_shutdown; -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] use std::collections::HashSet; pub type Sender = mpsc::UnboundedSender<(Instant, Arc)>; @@ -211,7 +213,7 @@ pub struct Connection { voice_call_request_timestamp: Option, audio_input_device_before_voice_call: Option, options_in_login: Option, - #[cfg(not(any(target_os = "android", target_os = "ios")))] + #[cfg(not(any(target_os = "ios")))] pressed_modifiers: HashSet, #[cfg(all(target_os = "linux", feature = "linux_headless"))] #[cfg(not(any(feature = "flatpak", feature = "appimage")))] @@ -348,7 +350,7 @@ impl Connection { voice_call_request_timestamp: None, audio_input_device_before_voice_call: None, options_in_login: None, - #[cfg(not(any(target_os = "android", target_os = "ios")))] + #[cfg(not(any(target_os = "ios")))] pressed_modifiers: Default::default(), #[cfg(all(target_os = "linux", feature = "linux_headless"))] #[cfg(not(any(feature = "flatpak", feature = "appimage")))] @@ -1726,6 +1728,12 @@ impl Connection { Some(message::Union::KeyEvent(..)) => {} #[cfg(any(target_os = "android"))] Some(message::Union::KeyEvent(mut me)) => { + let is_press = if cfg!(target_os = "linux") { + (me.press || me.down) && !crate::is_modifier(&me) + } else { + me.press + }; + let key = match me.mode.enum_value() { Ok(KeyboardMode::Map) => { Some(crate::keyboard::keycode_to_rdev_key(me.chr())) @@ -1738,7 +1746,28 @@ impl Connection { } } _ => None, - }; + } + .filter(crate::keyboard::is_modifier); + log::debug!("key:{:?}", key); + + if let Some(key) = key { + if is_press { + self.pressed_modifiers.insert(key); + } else { + self.pressed_modifiers.remove(&key); + } + } + + let mut modifiers = vec![]; + + for key in self.pressed_modifiers.iter() { + if let Some(control_key) = map_key_to_control_key(key) { + modifiers.push(EnumOrUnknown::new(control_key)); + } + } + + me.modifiers = modifiers; + let encode_result = me.write_to_bytes(); if let Ok(data) = encode_result {