From 22165ec1a50f236b0ed4eec854e70e95979c9fb5 Mon Sep 17 00:00:00 2001 From: mcfans Date: Thu, 19 Oct 2023 00:16:22 +0800 Subject: [PATCH] feat: legacy mode android keyboard support --- flutter/android/app/build.gradle | 28 +++++ .../com/carriez/flutter_hbb/InputService.kt | 92 ++++++++++++-- .../flutter_hbb/KeyboardKeyEventMapper.kt | 115 ++++++++++++++++++ .../com/carriez/flutter_hbb/MainService.kt | 5 +- .../res/xml/accessibility_service_config.xml | 1 + libs/scrap/src/android/ffi.rs | 11 +- src/keyboard.rs | 6 +- src/server/connection.rs | 31 +++-- 8 files changed, 261 insertions(+), 28 deletions(-) create mode 100644 flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index f4dc69e41..3793c1f4c 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -1,3 +1,8 @@ +import com.google.protobuf.gradle.* +plugins { + id "com.google.protobuf" version "0.9.4" +} + def keystoreProperties = new Properties() def keystorePropertiesFile = rootProject.file('key.properties') if (keystorePropertiesFile.exists()) { @@ -31,10 +36,33 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" +dependencies { + implementation 'com.google.protobuf:protobuf-javalite:3.20.1' +} + +protobuf { + protoc { + artifact = 'com.google.protobuf:protoc:3.20.1' + } + + generateProtoTasks { + all().configureEach { task -> + task.builtins { + java { + option "lite" + } + } + } + } +} + android { compileSdkVersion 33 sourceSets { main.java.srcDirs += 'src/main/kotlin' + + main.proto.srcDirs += '../../../libs/hbb_common/protos' + main.proto.includes += "message.proto" } compileOptions { diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index d46f084a3..50445589c 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -11,13 +11,18 @@ import android.accessibilityservice.GestureDescription import android.graphics.Path import android.os.Build import android.os.Bundle +import android.os.Handler +import android.os.Looper import android.util.Log +import android.widget.EditText import android.view.accessibility.AccessibilityEvent import android.view.accessibility.AccessibilityNodeInfo import androidx.annotation.RequiresApi import java.util.* import kotlin.math.abs import kotlin.math.max +import hbb.MessageOuterClass.KeyEvent; +import hbb.KeyEventConverter; const val LIFT_DOWN = 9 const val LIFT_MOVE = 8 @@ -60,6 +65,8 @@ class InputService : AccessibilityService() { private var isWheelActionsPolling = false private var isWaitingLongPress = false + private var fakeEditTextForTextStateCalculation: EditText? = null + @RequiresApi(Build.VERSION_CODES.N) fun onMouseInput(mask: Int, _x: Int, _y: Int) { val x = max(0, _x) @@ -255,20 +262,87 @@ class InputService : AccessibilityService() { } @RequiresApi(Build.VERSION_CODES.N) - fun onTextInput(str: String) { - findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { - val arguments = Bundle() - arguments.putCharSequence( - AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, - str - ) - it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + fun onKeyEvent(data: ByteArray) { + val keyEvent = KeyEvent.parseFrom(data); + val handler = Handler(Looper.getMainLooper()) + handler.post { + findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node -> + val text = node.getText() + val isShowingHint = node.isShowingHintText() + + var textSelectionStart = node.getTextSelectionStart() + var textSelectionEnd = node.getTextSelectionEnd() + + if (text != null) { + if (textSelectionStart > text.length) { + textSelectionStart = text.length + } + if (textSelectionEnd > text.length) { + textSelectionEnd = text.length + } + if (textSelectionStart > textSelectionEnd) { + textSelectionStart = textSelectionEnd + } + } + + if (keyEvent.hasSeq()) { + val seq = keyEvent.getSeq() + + var newText = "" + + if ((textSelectionStart == -1) || (textSelectionEnd == -1)) { + newText = seq + } else { + newText = text.let { + it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart) + } + } + + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText + ) + node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + + } else { + KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event -> + Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd") + if (isShowingHint) { + this.fakeEditTextForTextStateCalculation?.setText(null) + } else { + this.fakeEditTextForTextStateCalculation?.setText(text) + } + if (textSelectionStart != -1 && textSelectionEnd != -1) { + this.fakeEditTextForTextStateCalculation?.setSelection( + textSelectionStart, + textSelectionEnd + ) + } + this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event) + + this.fakeEditTextForTextStateCalculation?.getText()?.let { newText -> + val arguments = Bundle() + arguments.putCharSequence( + AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE, + newText.toString() + ) + node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments) + } + } + } + } } } + + override fun onAccessibilityEvent(event: AccessibilityEvent) { + } + override fun onServiceConnected() { super.onServiceConnected() ctx = this + fakeEditTextForTextStateCalculation = EditText(this) Log.d(logTag, "onServiceConnected!") } @@ -277,7 +351,5 @@ class InputService : AccessibilityService() { super.onDestroy() } - override fun onAccessibilityEvent(event: AccessibilityEvent?) {} - override fun onInterrupt() {} } diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt new file mode 100644 index 000000000..f7273625e --- /dev/null +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt @@ -0,0 +1,115 @@ +package hbb; +import android.view.KeyEvent +import android.view.KeyCharacterMap +import hbb.MessageOuterClass.ControlKey + +object KeyEventConverter { + fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent { + var chrValue = 0 + var modifiers = 0 + + android.util.Log.d(tag, "proto: $keyEventProto") + + if (keyEventProto.hasUnicode()) { + chrValue = + } + + if (keyEventProto.hasChr()) { + chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int) + } else if (keyEventProto.hasControlKey()) { + chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey()) + } + + var modifiersList = keyEventProto.getModifiersList() + + if (modifiersList != null) { + for (modifier in keyEventProto.getModifiersList()) { + val modifierValue = convertModifier(modifier) + modifiers = modifiers and modifierValue + } + } + + var action = 0 + if (keyEventProto.getDown()) { + action = KeyEvent.ACTION_DOWN + } else { + action = KeyEvent.ACTION_UP + } + + return KeyEvent(0, 0, action, chrValue, 0, modifiers) + } + + private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int { + // Add logic to map ControlKey values to Android KeyEvent key codes. + // You'll need to provide the mapping for each key. + return when (controlKey) { + ControlKey.Alt -> KeyEvent.META_ALT_ON + ControlKey.Control -> KeyEvent.META_CTRL_ON + ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON + ControlKey.Meta -> KeyEvent.META_META_ON + ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON + ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON + ControlKey.Shift -> KeyEvent.META_SHIFT_ON + ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON + ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON + else -> 0 // Default to unknown. + } + } + + private val tag = "KeyEventConverter" + + private fun convertUnicodeToKeyCode(unicode: Int): Int { + val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD) + android.util.Log.d(tag, "unicode: $unicode") + val events = charMap.getEvents(charArrayOf(unicode.toChar())) + if (events != null && events.size > 0) { + android.util.Log.d(tag, "keycode ${events[0].keyCode}") + return events[0].keyCode + } + return 0 + } + + private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int { + // Add logic to map ControlKey values to Android KeyEvent key codes. + // You'll need to provide the mapping for each key. + return when (controlKey) { + ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT + ControlKey.Backspace -> KeyEvent.KEYCODE_DEL + ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT + ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK + ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT + ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK + ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT + ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT + ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT + ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT + ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN + ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT + ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT + ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP + ControlKey.End -> KeyEvent.KEYCODE_MOVE_END + ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME + ControlKey.Insert -> KeyEvent.KEYCODE_INSERT + ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE + ControlKey.F1 -> KeyEvent.KEYCODE_F1 + ControlKey.F2 -> KeyEvent.KEYCODE_F2 + ControlKey.F3 -> KeyEvent.KEYCODE_F3 + ControlKey.F4 -> KeyEvent.KEYCODE_F4 + ControlKey.F5 -> KeyEvent.KEYCODE_F5 + ControlKey.F6 -> KeyEvent.KEYCODE_F6 + ControlKey.F7 -> KeyEvent.KEYCODE_F7 + ControlKey.F8 -> KeyEvent.KEYCODE_F8 + ControlKey.F9 -> KeyEvent.KEYCODE_F9 + ControlKey.F10 -> KeyEvent.KEYCODE_F10 + ControlKey.F11 -> KeyEvent.KEYCODE_F11 + ControlKey.F12 -> KeyEvent.KEYCODE_F12 + ControlKey.Space -> KeyEvent.KEYCODE_SPACE + ControlKey.Tab -> KeyEvent.KEYCODE_TAB + ControlKey.Return -> KeyEvent.KEYCODE_ENTER + ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL + ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR + ControlKey.Pause -> KeyEvent.KEYCODE_BREAK + else -> 0 // Default to unknown. + } + } +} diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index e9b9c8ef0..cc2e20e25 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -44,7 +44,6 @@ import java.nio.ByteBuffer import kotlin.math.max import kotlin.math.min - const val DEFAULT_NOTIFY_TITLE = "RustDesk" const val DEFAULT_NOTIFY_TEXT = "Service is running" const val DEFAULT_NOTIFY_ID = 1 @@ -96,8 +95,8 @@ class MainService : Service() { @Keep @RequiresApi(Build.VERSION_CODES.N) - fun rustInputString(input: String) { - InputService.ctx?.onTextInput(input) + fun rustKeyEventInput(input: ByteArray) { + InputService.ctx?.onKeyEvent(input) } @Keep diff --git a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml index fa9407128..90b57cd4e 100644 --- a/flutter/android/app/src/main/res/xml/accessibility_service_config.xml +++ b/flutter/android/app/src/main/res/xml/accessibility_service_config.xml @@ -1,5 +1,6 @@ } } -pub fn call_main_service_input_string(str: &str) -> JniResult<()> { +pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), MAIN_SERVICE_CTX.read().unwrap().as_ref(), ) { let mut env = jvm.attach_current_thread_as_daemon()?; - let input_string = env.new_string(str)?; + let data = env.byte_array_from_slice(data)?; + env.call_method( ctx, - "rustInputString", - "(Ljava/lang/String;)V", - &[JValue::Object(&JObject::from(input_string))], + "rustKeyEventInput", + "([B)V", + &[JValue::Object(&JObject::from(data))], )?; return Ok(()); } else { diff --git a/src/keyboard.rs b/src/keyboard.rs index 9a6ac49d5..46d0c8f9d 100644 --- a/src/keyboard.rs +++ b/src/keyboard.rs @@ -877,7 +877,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) -> Some(key_event) } -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec) { match &event.unicode { Some(unicode_info) => { @@ -1046,11 +1046,11 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) - events } -#[cfg(not(any(target_os = "android", target_os = "ios")))] +#[cfg(not(any(target_os = "ios")))] pub fn keycode_to_rdev_key(keycode: u32) -> Key { #[cfg(target_os = "windows")] return rdev::win_key_from_scancode(keycode); - #[cfg(target_os = "linux")] + #[cfg(any(target_os = "linux", target_os = "android"))] return rdev::linux_key_from_code(keycode); #[cfg(target_os = "macos")] return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default()); diff --git a/src/server/connection.rs b/src/server/connection.rs index 46e1e4298..20e2c9e2e 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -41,7 +41,7 @@ use hbb_common::{ tokio_util::codec::{BytesCodec, Framed}, }; #[cfg(any(target_os = "android", target_os = "ios"))] -use scrap::android::{call_main_service_pointer_input, call_main_service_input_string}; +use scrap::android::{call_main_service_pointer_input, call_main_service_key_event}; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1725,12 +1725,29 @@ impl Connection { #[cfg(any(target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} #[cfg(any(target_os = "android"))] - Some(message::Union::KeyEvent(me)) => { - // We can only use seq of key event, android device doesn't support abritrary key stroke. - let seq = me.seq(); - let result = call_main_service_input_string(seq); - if let Err(e) = result { - log::debug!("call_main_service_input_string fail:{}", e); + Some(message::Union::KeyEvent(mut me)) => { + let key = match me.mode.enum_value() { + Ok(KeyboardMode::Map) => { + Some(crate::keyboard::keycode_to_rdev_key(me.chr())) + } + Ok(KeyboardMode::Translate) => { + if let Some(key_event::Union::Chr(code)) = me.union { + Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF)) + } else { + None + } + } + _ => None, + }; + let encode_result = me.write_to_bytes(); + + if let Ok(data) = encode_result { + let result = call_main_service_key_event(&data); + if let Err(e) = result { + log::debug!("call_main_service_key_event fail:{}", e); + } + } else { + log::debug!("encode key event fail:{}", encode_result.err().unwrap()); } } #[cfg(not(any(target_os = "android", target_os = "ios")))]