feat: legacy mode android keyboard support

flutter/android/app/build.gradle

@@ -1,3 +1,8 @@
+import com.google.protobuf.gradle.*
+plugins {
+    id "com.google.protobuf" version "0.9.4"
+}
+
 def keystoreProperties = new Properties()
 def keystorePropertiesFile = rootProject.file('key.properties')
 if (keystorePropertiesFile.exists()) {
@@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
 
+dependencies {
+    implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
+}
+
+protobuf {
+    protoc {
+        artifact = 'com.google.protobuf:protoc:3.20.1'
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}
+
 android {
     compileSdkVersion 33
     sourceSets {
         main.java.srcDirs += 'src/main/kotlin'
+
+        main.proto.srcDirs += '../../../libs/hbb_common/protos'
+        main.proto.includes += "message.proto"
     }
 
     compileOptions {

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -11,13 +11,18 @@ import android.accessibilityservice.GestureDescription
 import android.graphics.Path
 import android.os.Build
 import android.os.Bundle
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
+import android.widget.EditText
 import android.view.accessibility.AccessibilityEvent
 import android.view.accessibility.AccessibilityNodeInfo
 import androidx.annotation.RequiresApi
 import java.util.*
 import kotlin.math.abs
 import kotlin.math.max
+import hbb.MessageOuterClass.KeyEvent;
+import hbb.KeyEventConverter;
 
 const val LIFT_DOWN = 9
 const val LIFT_MOVE = 8
@@ -60,6 +65,8 @@ class InputService : AccessibilityService() {
     private var isWheelActionsPolling = false
     private var isWaitingLongPress = false
 
+    private var fakeEditTextForTextStateCalculation: EditText? = null
+
     @RequiresApi(Build.VERSION_CODES.N)
     fun onMouseInput(mask: Int, _x: Int, _y: Int) {
         val x = max(0, _x)
@@ -255,20 +262,87 @@ class InputService : AccessibilityService() {
     }
 
     @RequiresApi(Build.VERSION_CODES.N)
-    fun onTextInput(str: String) {
+    fun onKeyEvent(data: ByteArray) {
-        findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let {
+        val keyEvent = KeyEvent.parseFrom(data);
-            val arguments = Bundle()
+        val handler = Handler(Looper.getMainLooper())
-            arguments.putCharSequence(
+        handler.post {
-                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+            findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let { node ->
-                str
+                val text = node.getText()
-            )
+                val isShowingHint = node.isShowingHintText()
-            it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+
+                var textSelectionStart = node.getTextSelectionStart()
+                var textSelectionEnd = node.getTextSelectionEnd()
+
+                if (text != null) {
+                    if (textSelectionStart > text.length) {
+                        textSelectionStart = text.length
+                    }
+                    if (textSelectionEnd > text.length) {
+                        textSelectionEnd = text.length
+                    }
+                    if (textSelectionStart > textSelectionEnd) {
+                        textSelectionStart = textSelectionEnd
+                    }
+                }
+
+                if (keyEvent.hasSeq()) {
+                    val seq = keyEvent.getSeq()
+
+                    var newText = ""
+
+                    if ((textSelectionStart == -1) || (textSelectionEnd == -1)) {
+                        newText = seq
+                    } else {
+                        newText = text.let {
+                            it.substring(0, textSelectionStart) + seq + it.substring(textSelectionStart)
+                        }
+                    }
+
+                    val arguments = Bundle()
+                    arguments.putCharSequence(
+                        AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                        newText
+                    )
+                    node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+
+                } else {
+                    KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
+                        Log.d(logTag, "event $event text $text start $textSelectionStar end $textSelectionEnd")
+                        if (isShowingHint) {
+                            this.fakeEditTextForTextStateCalculation?.setText(null)
+                        } else {
+                            this.fakeEditTextForTextStateCalculation?.setText(text)
+                        }
+                        if (textSelectionStart != -1 && textSelectionEnd != -1) {
+                            this.fakeEditTextForTextStateCalculation?.setSelection(
+                                textSelectionStart,
+                                textSelectionEnd
+                            )
+                        }
+                        this.fakeEditTextForTextStateCalculation?.dispatchKeyEvent(event)
+
+                        this.fakeEditTextForTextStateCalculation?.getText()?.let { newText ->
+                            val arguments = Bundle()
+                            arguments.putCharSequence(
+                                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                                newText.toString()
+                            )
+                            node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+                        }
+                    }
+                }
+            }
         }
     }
 
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent) {
+    }
+
     override fun onServiceConnected() {
         super.onServiceConnected()
         ctx = this
+        fakeEditTextForTextStateCalculation = EditText(this)
         Log.d(logTag, "onServiceConnected!")
     }
 
@@ -277,7 +351,5 @@ class InputService : AccessibilityService() {
         super.onDestroy()
     }
 
-    override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
-
     override fun onInterrupt() {}
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt

@@ -0,0 +1,115 @@
+package hbb;
+import android.view.KeyEvent
+import android.view.KeyCharacterMap
+import hbb.MessageOuterClass.ControlKey
+
+object KeyEventConverter {
+    fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
+        var chrValue = 0
+        var modifiers = 0
+
+        android.util.Log.d(tag, "proto: $keyEventProto")
+
+        if (keyEventProto.hasUnicode()) {
+            chrValue = 
+        }
+
+        if (keyEventProto.hasChr()) {
+            chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+        } else if (keyEventProto.hasControlKey()) {
+            chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
+        }
+
+        var modifiersList = keyEventProto.getModifiersList()
+
+        if (modifiersList != null) {
+            for (modifier in keyEventProto.getModifiersList()) {
+                val modifierValue = convertModifier(modifier)
+                modifiers = modifiers and modifierValue
+            }
+        }
+
+        var action = 0
+        if (keyEventProto.getDown()) {
+            action = KeyEvent.ACTION_DOWN
+        } else {
+            action = KeyEvent.ACTION_UP
+        }
+
+        return KeyEvent(0, 0, action, chrValue, 0, modifiers)
+    }
+
+    private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.META_ALT_ON
+            ControlKey.Control -> KeyEvent.META_CTRL_ON
+            ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
+            ControlKey.Meta -> KeyEvent.META_META_ON
+            ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
+            ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
+            ControlKey.Shift -> KeyEvent.META_SHIFT_ON
+            ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
+            ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
+            else -> 0 // Default to unknown.
+        }
+    }
+
+    private val tag = "KeyEventConverter"
+
+    private fun convertUnicodeToKeyCode(unicode: Int): Int {
+        val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+        android.util.Log.d(tag, "unicode: $unicode")
+        val events = charMap.getEvents(charArrayOf(unicode.toChar()))
+        if (events != null && events.size > 0) {
+            android.util.Log.d(tag, "keycode ${events[0].keyCode}")
+            return events[0].keyCode
+        }
+        return 0
+    }
+
+    private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
+            ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
+            ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
+            ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
+            ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
+            ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
+            ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
+            ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
+            ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
+            ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
+            ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
+            ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
+            ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
+            ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
+            ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
+            ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
+            ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
+            ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
+            ControlKey.F1 -> KeyEvent.KEYCODE_F1
+            ControlKey.F2 -> KeyEvent.KEYCODE_F2
+            ControlKey.F3 -> KeyEvent.KEYCODE_F3
+            ControlKey.F4 -> KeyEvent.KEYCODE_F4
+            ControlKey.F5 -> KeyEvent.KEYCODE_F5
+            ControlKey.F6 -> KeyEvent.KEYCODE_F6
+            ControlKey.F7 -> KeyEvent.KEYCODE_F7
+            ControlKey.F8 -> KeyEvent.KEYCODE_F8
+            ControlKey.F9 -> KeyEvent.KEYCODE_F9
+            ControlKey.F10 -> KeyEvent.KEYCODE_F10
+            ControlKey.F11 -> KeyEvent.KEYCODE_F11
+            ControlKey.F12 -> KeyEvent.KEYCODE_F12
+            ControlKey.Space -> KeyEvent.KEYCODE_SPACE
+            ControlKey.Tab -> KeyEvent.KEYCODE_TAB
+            ControlKey.Return -> KeyEvent.KEYCODE_ENTER
+            ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
+            ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
+            ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
+            else -> 0 // Default to unknown.
+        }
+    }
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -44,7 +44,6 @@ import java.nio.ByteBuffer
 import kotlin.math.max
 import kotlin.math.min
 
-
 const val DEFAULT_NOTIFY_TITLE = "RustDesk"
 const val DEFAULT_NOTIFY_TEXT = "Service is running"
 const val DEFAULT_NOTIFY_ID = 1
@@ -96,8 +95,8 @@ class MainService : Service() {
 
     @Keep
     @RequiresApi(Build.VERSION_CODES.N)
-    fun rustInputString(input: String) {
+    fun rustKeyEventInput(input: ByteArray) {
-        InputService.ctx?.onTextInput(input)
+        InputService.ctx?.onKeyEvent(input)
     }
 
     @Keep

flutter/android/app/src/main/res/xml/accessibility_service_config.xml

@@ -1,5 +1,6 @@
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
     android:accessibilityEventTypes="typeWindowsChanged"
+    android:canRetrieveWindowContent="true"
     android:accessibilityFlags="flagDefault"
     android:notificationTimeout="50"
     android:description="@string/accessibility_service_description"

libs/scrap/src/android/ffi.rs

@@ -173,18 +173,19 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) ->
     }
 }
 
-pub fn call_main_service_input_string(str: &str) -> JniResult<()> {
+pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> {
     if let (Some(jvm), Some(ctx)) = (
         JVM.read().unwrap().as_ref(),
         MAIN_SERVICE_CTX.read().unwrap().as_ref(),
     ) {
         let mut env = jvm.attach_current_thread_as_daemon()?;
-        let input_string = env.new_string(str)?;
+        let data = env.byte_array_from_slice(data)?;
+
         env.call_method(
             ctx,
-            "rustInputString",
+            "rustKeyEventInput",
-            "(Ljava/lang/String;)V",
+            "([B)V",
-            &[JValue::Object(&JObject::from(input_string))],
+            &[JValue::Object(&JObject::from(data))],
         )?;
         return Ok(());
     } else {

src/keyboard.rs

@@ -877,7 +877,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) ->
     Some(key_event)
 }
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec<KeyEvent>) {
     match &event.unicode {
         Some(unicode_info) => {
@@ -1046,11 +1046,11 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) -
     events
 }
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 pub fn keycode_to_rdev_key(keycode: u32) -> Key {
     #[cfg(target_os = "windows")]
     return rdev::win_key_from_scancode(keycode);
-    #[cfg(target_os = "linux")]
+    #[cfg(any(target_os = "linux", target_os = "android"))]
     return rdev::linux_key_from_code(keycode);
     #[cfg(target_os = "macos")]
     return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default());

src/server/connection.rs

@@ -41,7 +41,7 @@ use hbb_common::{
     tokio_util::codec::{BytesCodec, Framed},
 };
 #[cfg(any(target_os = "android", target_os = "ios"))]
-use scrap::android::{call_main_service_pointer_input, call_main_service_input_string};
+use scrap::android::{call_main_service_pointer_input, call_main_service_key_event};
 use serde_json::{json, value::Value};
 use sha2::{Digest, Sha256};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -1725,12 +1725,29 @@ impl Connection {
                 #[cfg(any(target_os = "ios"))]
                 Some(message::Union::KeyEvent(..)) => {}
                 #[cfg(any(target_os = "android"))]
-                Some(message::Union::KeyEvent(me)) => {
+                Some(message::Union::KeyEvent(mut me)) => {
-                    // We can only use seq of key event, android device doesn't support abritrary key stroke.
+                    let key = match me.mode.enum_value() {
-                    let seq = me.seq();
+                        Ok(KeyboardMode::Map) => {
-                    let result = call_main_service_input_string(seq);
+                            Some(crate::keyboard::keycode_to_rdev_key(me.chr()))
-                    if let Err(e) = result {
+                        }
-                        log::debug!("call_main_service_input_string fail:{}", e);
+                        Ok(KeyboardMode::Translate) => {
+                            if let Some(key_event::Union::Chr(code)) = me.union {
+                                Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF))
+                            } else {
+                                None
+                            }
+                        }
+                        _ => None,
+                    };
+                    let encode_result = me.write_to_bytes();
+
+                    if let Ok(data) = encode_result {
+                        let result = call_main_service_key_event(&data);
+                        if let Err(e) = result {
+                            log::debug!("call_main_service_key_event fail:{}", e);
+                        }
+                    } else {
+                        log::debug!("encode key event fail:{}", encode_result.err().unwrap());
                     }
                 }
                 #[cfg(not(any(target_os = "android", target_os = "ios")))]