Merge pull request #6097 from mcfans/master

Physical keyboard to android support

Cargo.lock

@@ -1796,7 +1796,7 @@ dependencies = [
  "log",
  "objc",
  "pkg-config",
- "rdev",
+ "rdev 0.5.0-2 (git+https://github.com/fufesou/rdev)",
  "serde 1.0.190",
  "serde_derive",
  "tfc",
@@ -4881,6 +4881,30 @@ dependencies = [
  "crossbeam-utils",
 ]
 
+[[package]]
+name = "rdev"
+version = "0.5.0-2"
+source = "git+https://github.com/fufesou/rdev?branch=master#339b2a334ba273afebb7e27fb76984e620fc76e5"
+dependencies = [
+ "cocoa",
+ "core-foundation 0.9.3 (registry+https://github.com/rust-lang/crates.io-index)",
+ "core-foundation-sys 0.8.4",
+ "core-graphics 0.22.3",
+ "dispatch",
+ "enum-map",
+ "epoll",
+ "inotify",
+ "lazy_static",
+ "libc",
+ "log",
+ "mio",
+ "strum 0.24.1",
+ "strum_macros 0.24.3",
+ "widestring",
+ "winapi 0.3.9",
+ "x11 2.21.0",
+]
+
 [[package]]
 name = "rdev"
 version = "0.5.0-2"
@@ -5226,7 +5250,7 @@ dependencies = [
  "pam",
  "parity-tokio-ipc",
  "percent-encoding",
- "rdev",
+ "rdev 0.5.0-2 (git+https://github.com/fufesou/rdev?branch=master)",
  "repng",
  "reqwest",
  "ringbuf",

Cargo.toml

@@ -71,7 +71,7 @@ default-net = "0.14"
 wol-rs = "1.0"
 flutter_rust_bridge = { version = "=1.80", features = ["uuid"], optional = true}
 errno = "0.3"
-rdev = { git = "https://github.com/fufesou/rdev" }
+rdev = { git = "https://github.com/fufesou/rdev", branch = "master" }
 url = { version = "2.3", features = ["serde"] }
 crossbeam-queue = "0.3"
 hex = "0.4"

flutter/android/app/build.gradle

@@ -1,3 +1,8 @@
+import com.google.protobuf.gradle.*
+plugins {
+    id "com.google.protobuf" version "0.9.4"
+}
+
 def keystoreProperties = new Properties()
 def keystorePropertiesFile = rootProject.file('key.properties')
 if (keystorePropertiesFile.exists()) {
@@ -31,10 +36,33 @@ apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
 apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
 
+dependencies {
+    implementation 'com.google.protobuf:protobuf-javalite:3.20.1'
+}
+
+protobuf {
+    protoc {
+        artifact = 'com.google.protobuf:protoc:3.20.1'
+    }
+
+    generateProtoTasks {
+        all().configureEach { task ->
+            task.builtins {
+                java {
+                    option "lite"
+                }
+            }
+        }
+    }
+}
+
 android {
     compileSdkVersion 33
     sourceSets {
         main.java.srcDirs += 'src/main/kotlin'
+
+        main.proto.srcDirs += '../../../libs/hbb_common/protos'
+        main.proto.includes += "message.proto"
     }
 
     compileOptions {
@@ -65,6 +93,7 @@ android {
             // TODO: Add your own signing config for the release build.
             // Signing with the debug keys for now, so `flutter run --release` works.
             signingConfig signingConfigs.release
+            proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules'
         }
     }
 }

flutter/android/app/proguard-rules

@@ -0,0 +1,4 @@
+# Keep class members from protobuf generated code.
+-keepclassmembers class * extends com.google.protobuf.GeneratedMessageLite {
+  <fields>;
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -10,12 +10,27 @@ import android.accessibilityservice.AccessibilityService
 import android.accessibilityservice.GestureDescription
 import android.graphics.Path
 import android.os.Build
+import android.os.Bundle
+import android.os.Handler
+import android.os.Looper
 import android.util.Log
+import android.widget.EditText
 import android.view.accessibility.AccessibilityEvent
+import android.view.ViewGroup.LayoutParams
+import android.view.accessibility.AccessibilityNodeInfo
+import android.graphics.Rect
+import android.accessibilityservice.AccessibilityServiceInfo
+import android.accessibilityservice.AccessibilityServiceInfo.FLAG_INPUT_METHOD_EDITOR
+import android.accessibilityservice.AccessibilityServiceInfo.FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+import android.view.inputmethod.EditorInfo
 import androidx.annotation.RequiresApi
 import java.util.*
+import java.lang.Character
 import kotlin.math.abs
 import kotlin.math.max
+import hbb.MessageOuterClass.KeyEvent
+import hbb.MessageOuterClass.KeyboardMode
+import hbb.KeyEventConverter
 
 const val LIFT_DOWN = 9
 const val LIFT_MOVE = 8
@@ -58,6 +73,8 @@ class InputService : AccessibilityService() {
     private var isWheelActionsPolling = false
     private var isWaitingLongPress = false
 
+    private var fakeEditTextForTextStateCalculation: EditText? = null
+
     @RequiresApi(Build.VERSION_CODES.N)
     fun onMouseInput(mask: Int, _x: Int, _y: Int) {
         val x = max(0, _x)
@@ -252,9 +269,296 @@ class InputService : AccessibilityService() {
         }
     }
 
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun onKeyEvent(data: ByteArray) {
+        val keyEvent = KeyEvent.parseFrom(data)
+        val keyboardMode = keyEvent.getMode()
+
+        var textToCommit: String? = null
+
+        if (keyboardMode == KeyboardMode.Legacy) {
+            if (keyEvent.hasChr() && keyEvent.getDown()) {
+                val chr = keyEvent.getChr()
+                if (chr != null) {
+                    textToCommit = String(Character.toChars(chr))
+                }
+            }
+        } else if (keyboardMode == KeyboardMode.Translate) {
+            if (keyEvent.hasSeq() && keyEvent.getDown()) {
+                val seq = keyEvent.getSeq()
+                if (seq != null) {
+                    textToCommit = seq
+                }
+            }
+        }
+
+        Log.d(logTag, "onKeyEvent $keyEvent textToCommit:$textToCommit")
+
+        if (Build.VERSION.SDK_INT >= 33) {
+            getInputMethod()?.let { inputMethod ->
+                inputMethod.getCurrentInputConnection()?.let { inputConnection ->
+                    if (textToCommit != null) {
+                        textToCommit?.let { text ->
+                            inputConnection.commitText(text, 1, null)
+                        }
+                    } else {
+                        KeyEventConverter.toAndroidKeyEvent(keyEvent).let { event ->
+                            inputConnection.sendKeyEvent(event)
+                        }
+                    }
+                }
+            }
+        } else {
+            val handler = Handler(Looper.getMainLooper())
+            handler.post {
+                KeyEventConverter.toAndroidKeyEvent(keyEvent)?.let { event ->
+                    val possibleNodes = possibleAccessibiltyNodes()
+                    Log.d(logTag, "possibleNodes:$possibleNodes")
+                    for (item in possibleNodes) {
+                        val success = trySendKeyEvent(event, item, textToCommit)
+                        if (success) {
+                            break
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private fun insertAccessibilityNode(list: LinkedList<AccessibilityNodeInfo>, node: AccessibilityNodeInfo) {
+        if (node == null) {
+            return
+        }
+        if (list.contains(node)) {
+            return
+        }
+        list.add(node)
+    }
+
+    private fun findChildNode(node: AccessibilityNodeInfo?): AccessibilityNodeInfo? {
+        if (node == null) {
+            return null
+        }
+        if (node.isEditable() && node.isFocusable()) {
+            return node
+        }
+        val childCount = node.getChildCount()
+        for (i in 0 until childCount) {
+            val child = node.getChild(i)
+            if (child != null) {
+                if (child.isEditable() && child.isFocusable()) {
+                    return child
+                }
+                if (Build.VERSION.SDK_INT < 33) {
+                    child.recycle()
+                }
+            }
+        }
+        for (i in 0 until childCount) {
+            val child = node.getChild(i)
+            if (child != null) {
+                val result = findChildNode(child)
+                if (Build.VERSION.SDK_INT < 33) {
+                    if (child != result) {
+                        child.recycle()
+                    }
+                }
+                if (result != null) {
+                    return result
+                }
+            }
+        }
+        return null
+    }
+
+    private fun possibleAccessibiltyNodes(): LinkedList<AccessibilityNodeInfo> {
+        val linkedList = LinkedList<AccessibilityNodeInfo>()
+        val latestList = LinkedList<AccessibilityNodeInfo>()
+
+        val focusInput = findFocus(AccessibilityNodeInfo.FOCUS_INPUT)
+        var focusAccessibilityInput = findFocus(AccessibilityNodeInfo.FOCUS_ACCESSIBILITY)
+
+        val rootInActiveWindow = getRootInActiveWindow()
+
+        Log.d(logTag, "focusInput:$focusInput focusAccessibilityInput:$focusAccessibilityInput rootInActiveWindow:$rootInActiveWindow")
+
+        if (focusInput != null) {
+            if (focusInput.isFocusable() && focusInput.isEditable()) {
+                insertAccessibilityNode(linkedList, focusInput)
+            } else {
+                insertAccessibilityNode(latestList, focusInput)
+            }
+        }
+
+        if (focusAccessibilityInput != null) {
+            if (focusAccessibilityInput.isFocusable() && focusAccessibilityInput.isEditable()) {
+                insertAccessibilityNode(linkedList, focusAccessibilityInput)
+            } else {
+                insertAccessibilityNode(latestList, focusAccessibilityInput)
+            }
+        }
+
+        val childFromFocusInput = findChildNode(focusInput)
+        Log.d(logTag, "childFromFocusInput:$childFromFocusInput")
+
+        if (childFromFocusInput != null) {
+            insertAccessibilityNode(linkedList, childFromFocusInput)
+        }
+
+        val childFromFocusAccessibilityInput = findChildNode(focusAccessibilityInput)
+        if (childFromFocusAccessibilityInput != null) {
+            insertAccessibilityNode(linkedList, childFromFocusAccessibilityInput)
+        }
+        Log.d(logTag, "childFromFocusAccessibilityInput:$childFromFocusAccessibilityInput")
+
+        if (rootInActiveWindow != null) {
+            insertAccessibilityNode(linkedList, rootInActiveWindow)
+        }
+
+        for (item in latestList) {
+            insertAccessibilityNode(linkedList, item)
+        }
+
+        return linkedList
+    }
+
+    private fun trySendKeyEvent(event: android.view.KeyEvent, node: AccessibilityNodeInfo, textToCommit: String?): Boolean {
+        node.refresh()
+        this.fakeEditTextForTextStateCalculation?.setSelection(0,0)
+        this.fakeEditTextForTextStateCalculation?.setText(null)
+
+        val text = node.getText()
+        var isShowingHint = false
+        if (Build.VERSION.SDK_INT >= 26) {
+            isShowingHint = node.isShowingHintText()
+        }
+
+        var textSelectionStart = node.textSelectionStart
+        var textSelectionEnd = node.textSelectionEnd
+
+        if (text != null) {
+            if (textSelectionStart > text.length) {
+                textSelectionStart = text.length
+            }
+            if (textSelectionEnd > text.length) {
+                textSelectionEnd = text.length
+            }
+            if (textSelectionStart > textSelectionEnd) {
+                textSelectionStart = textSelectionEnd
+            }
+        }
+
+        var success = false
+
+        Log.d(logTag, "existing text:$text textToCommit:$textToCommit textSelectionStart:$textSelectionStart textSelectionEnd:$textSelectionEnd")
+
+        if (textToCommit != null) {
+            if ((textSelectionStart == -1) || (textSelectionEnd == -1)) {
+                val newText = textToCommit
+                this.fakeEditTextForTextStateCalculation?.setText(newText)
+                success = updateTextForAccessibilityNode(node)
+            } else if (text != null) {
+                this.fakeEditTextForTextStateCalculation?.setText(text)
+                this.fakeEditTextForTextStateCalculation?.setSelection(
+                    textSelectionStart,
+                    textSelectionEnd
+                )
+                this.fakeEditTextForTextStateCalculation?.text?.insert(textSelectionStart, textToCommit)
+                success = updateTextAndSelectionForAccessibiltyNode(node)
+            }
+        } else {
+            if (isShowingHint) {
+                this.fakeEditTextForTextStateCalculation?.setText(null)
+            } else {
+                this.fakeEditTextForTextStateCalculation?.setText(text)
+            }
+            if (textSelectionStart != -1 && textSelectionEnd != -1) {
+                Log.d(logTag, "setting selection $textSelectionStart $textSelectionEnd")
+                this.fakeEditTextForTextStateCalculation?.setSelection(
+                    textSelectionStart,
+                    textSelectionEnd
+                )
+            }
+
+            this.fakeEditTextForTextStateCalculation?.let {
+                // This is essiential to make sure layout object is created. OnKeyDown may not work if layout is not created.
+                val rect = Rect()
+                node.getBoundsInScreen(rect)
+
+                it.layout(rect.left, rect.top, rect.right, rect.bottom)
+                it.onPreDraw()
+                if (event.action == android.view.KeyEvent.ACTION_DOWN) {
+                    val succ = it.onKeyDown(event.getKeyCode(), event)
+                    Log.d(logTag, "onKeyDown $succ")
+                } else if (event.action == android.view.KeyEvent.ACTION_UP) {
+                    val success = it.onKeyUp(event.getKeyCode(), event)
+                    Log.d(logTag, "keyup $success")
+                } else {}
+            }
+
+            success = updateTextAndSelectionForAccessibiltyNode(node)
+        }
+        return success
+    }
+
+    fun updateTextForAccessibilityNode(node: AccessibilityNodeInfo): Boolean {
+        var success = false
+        this.fakeEditTextForTextStateCalculation?.text?.let {
+            val arguments = Bundle()
+            arguments.putCharSequence(
+                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                it.toString()
+            )
+            success = node.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+        }
+        return success
+    }
+
+    fun updateTextAndSelectionForAccessibiltyNode(node: AccessibilityNodeInfo): Boolean {
+        var success = updateTextForAccessibilityNode(node)
+
+        if (success) {
+            val selectionStart = this.fakeEditTextForTextStateCalculation?.selectionStart
+            val selectionEnd = this.fakeEditTextForTextStateCalculation?.selectionEnd
+
+            if (selectionStart != null && selectionEnd != null) {
+                val arguments = Bundle()
+                arguments.putInt(
+                    AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_START_INT,
+                    selectionStart
+                )
+                arguments.putInt(
+                    AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT,
+                    selectionEnd
+                )
+                success = node.performAction(AccessibilityNodeInfo.ACTION_SET_SELECTION, arguments)
+                Log.d(logTag, "Update selection to $selectionStart $selectionEnd success:$success")
+            }
+        }
+
+        return success
+    }
+
+
+    override fun onAccessibilityEvent(event: AccessibilityEvent) {
+    }
+
     override fun onServiceConnected() {
         super.onServiceConnected()
         ctx = this
+        val info = AccessibilityServiceInfo()
+        if (Build.VERSION.SDK_INT >= 33) {
+            info.flags = FLAG_INPUT_METHOD_EDITOR or FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+        } else {
+            info.flags = FLAG_RETRIEVE_INTERACTIVE_WINDOWS
+        }
+        setServiceInfo(info)
+        fakeEditTextForTextStateCalculation = EditText(this)
+        // Size here doesn't matter, we won't show this view.
+        fakeEditTextForTextStateCalculation?.layoutParams = LayoutParams(100, 100)
+        fakeEditTextForTextStateCalculation?.onPreDraw()
+        val layout = fakeEditTextForTextStateCalculation?.getLayout()
+        Log.d(logTag, "fakeEditTextForTextStateCalculation layout:$layout")
         Log.d(logTag, "onServiceConnected!")
     }
 
@@ -263,7 +567,5 @@ class InputService : AccessibilityService() {
         super.onDestroy()
     }
 
-    override fun onAccessibilityEvent(event: AccessibilityEvent?) {}
-
     override fun onInterrupt() {}
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/KeyboardKeyEventMapper.kt

@@ -0,0 +1,118 @@
+package hbb;
+import android.view.KeyEvent
+import android.view.KeyCharacterMap
+import hbb.MessageOuterClass.KeyboardMode
+import hbb.MessageOuterClass.ControlKey
+
+object KeyEventConverter {
+    fun toAndroidKeyEvent(keyEventProto: hbb.MessageOuterClass.KeyEvent): KeyEvent {
+        var chrValue = 0
+        var modifiers = 0
+
+        val keyboardMode = keyEventProto.getMode()
+
+        if (keyEventProto.hasChr()) {
+            if (keyboardMode == KeyboardMode.Map || keyboardMode == KeyboardMode.Translate) {
+                chrValue = keyEventProto.getChr()
+            } else {
+                chrValue = convertUnicodeToKeyCode(keyEventProto.getChr() as Int)
+            }
+        } else if (keyEventProto.hasControlKey()) {
+            chrValue = convertControlKeyToKeyCode(keyEventProto.getControlKey())
+        }
+
+        var modifiersList = keyEventProto.getModifiersList()
+
+        if (modifiersList != null) {
+            for (modifier in keyEventProto.getModifiersList()) {
+                val modifierValue = convertModifier(modifier)
+                modifiers = modifiers or modifierValue
+            }
+        }
+
+        var action = 0
+        if (keyEventProto.getDown()) {
+            action = KeyEvent.ACTION_DOWN
+        } else {
+            action = KeyEvent.ACTION_UP
+        }
+
+        return KeyEvent(0, 0, action, chrValue, 0, modifiers)
+    }
+
+    private fun convertModifier(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.META_ALT_ON
+            ControlKey.Control -> KeyEvent.META_CTRL_ON
+            ControlKey.CapsLock -> KeyEvent.META_CAPS_LOCK_ON
+            ControlKey.Meta -> KeyEvent.META_META_ON
+            ControlKey.NumLock -> KeyEvent.META_NUM_LOCK_ON
+            ControlKey.RShift -> KeyEvent.META_SHIFT_RIGHT_ON
+            ControlKey.Shift -> KeyEvent.META_SHIFT_ON
+            ControlKey.RAlt -> KeyEvent.META_ALT_RIGHT_ON
+            ControlKey.RControl -> KeyEvent.META_CTRL_RIGHT_ON
+            else -> 0 // Default to unknown.
+        }
+    }
+
+    private val tag = "KeyEventConverter"
+
+    private fun convertUnicodeToKeyCode(unicode: Int): Int {
+        val charMap = KeyCharacterMap.load(KeyCharacterMap.VIRTUAL_KEYBOARD)
+        android.util.Log.d(tag, "unicode: $unicode")
+        val events = charMap.getEvents(charArrayOf(unicode.toChar()))
+        if (events != null && events.size > 0) {
+            android.util.Log.d(tag, "keycode ${events[0].keyCode}")
+            return events[0].keyCode
+        }
+        return 0
+    }
+
+    private fun convertControlKeyToKeyCode(controlKey: hbb.MessageOuterClass.ControlKey): Int {
+        // Add logic to map ControlKey values to Android KeyEvent key codes.
+        // You'll need to provide the mapping for each key.
+        return when (controlKey) {
+            ControlKey.Alt -> KeyEvent.KEYCODE_ALT_LEFT
+            ControlKey.Backspace -> KeyEvent.KEYCODE_DEL
+            ControlKey.Control -> KeyEvent.KEYCODE_CTRL_LEFT
+            ControlKey.CapsLock -> KeyEvent.KEYCODE_CAPS_LOCK
+            ControlKey.Meta -> KeyEvent.KEYCODE_META_LEFT
+            ControlKey.NumLock -> KeyEvent.KEYCODE_NUM_LOCK
+            ControlKey.RShift -> KeyEvent.KEYCODE_SHIFT_RIGHT
+            ControlKey.Shift -> KeyEvent.KEYCODE_SHIFT_LEFT
+            ControlKey.RAlt -> KeyEvent.KEYCODE_ALT_RIGHT
+            ControlKey.RControl -> KeyEvent.KEYCODE_CTRL_RIGHT
+            ControlKey.DownArrow -> KeyEvent.KEYCODE_DPAD_DOWN
+            ControlKey.LeftArrow -> KeyEvent.KEYCODE_DPAD_LEFT
+            ControlKey.RightArrow -> KeyEvent.KEYCODE_DPAD_RIGHT
+            ControlKey.UpArrow -> KeyEvent.KEYCODE_DPAD_UP
+            ControlKey.End -> KeyEvent.KEYCODE_MOVE_END
+            ControlKey.Home -> KeyEvent.KEYCODE_MOVE_HOME
+            ControlKey.PageUp -> KeyEvent.KEYCODE_PAGE_UP
+            ControlKey.PageDown -> KeyEvent.KEYCODE_PAGE_DOWN
+            ControlKey.Insert -> KeyEvent.KEYCODE_INSERT
+            ControlKey.Escape -> KeyEvent.KEYCODE_ESCAPE
+            ControlKey.F1 -> KeyEvent.KEYCODE_F1
+            ControlKey.F2 -> KeyEvent.KEYCODE_F2
+            ControlKey.F3 -> KeyEvent.KEYCODE_F3
+            ControlKey.F4 -> KeyEvent.KEYCODE_F4
+            ControlKey.F5 -> KeyEvent.KEYCODE_F5
+            ControlKey.F6 -> KeyEvent.KEYCODE_F6
+            ControlKey.F7 -> KeyEvent.KEYCODE_F7
+            ControlKey.F8 -> KeyEvent.KEYCODE_F8
+            ControlKey.F9 -> KeyEvent.KEYCODE_F9
+            ControlKey.F10 -> KeyEvent.KEYCODE_F10
+            ControlKey.F11 -> KeyEvent.KEYCODE_F11
+            ControlKey.F12 -> KeyEvent.KEYCODE_F12
+            ControlKey.Space -> KeyEvent.KEYCODE_SPACE
+            ControlKey.Tab -> KeyEvent.KEYCODE_TAB
+            ControlKey.Return -> KeyEvent.KEYCODE_ENTER
+            ControlKey.Delete -> KeyEvent.KEYCODE_FORWARD_DEL
+            ControlKey.Clear -> KeyEvent.KEYCODE_CLEAR
+            ControlKey.Pause -> KeyEvent.KEYCODE_BREAK
+            else -> 0 // Default to unknown.
+        }
+    }
+}

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -44,7 +44,6 @@ import java.nio.ByteBuffer
 import kotlin.math.max
 import kotlin.math.min
 
-
 const val DEFAULT_NOTIFY_TITLE = "RustDesk"
 const val DEFAULT_NOTIFY_TEXT = "Service is running"
 const val DEFAULT_NOTIFY_ID = 1
@@ -94,6 +93,12 @@ class MainService : Service() {
         }
     }
 
+    @Keep
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun rustKeyEventInput(input: ByteArray) {
+        InputService.ctx?.onKeyEvent(input)
+    }
+
     @Keep
     fun rustGetByName(name: String): String {
         return when (name) {

flutter/android/app/src/main/res/xml/accessibility_service_config.xml

@@ -1,5 +1,6 @@
 <accessibility-service xmlns:android="http://schemas.android.com/apk/res/android"
     android:accessibilityEventTypes="typeWindowsChanged"
+    android:canRetrieveWindowContent="true"
     android:accessibilityFlags="flagDefault"
     android:notificationTimeout="50"
     android:description="@string/accessibility_service_description"

flutter/lib/desktop/screen/desktop_remote_screen.dart

@@ -12,9 +12,9 @@ class DesktopRemoteScreen extends StatelessWidget {
   final Map<String, dynamic> params;
 
   DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) {
-    if (!bind.mainStartGrabKeyboard()) {
-      stateGlobal.grabKeyboard = true;
-    }
+      if (!bind.mainStartGrabKeyboard()) {
+        stateGlobal.grabKeyboard = true;
+      }
   }
 
   @override

flutter/lib/mobile/pages/remote_page.dart

@@ -364,6 +364,10 @@ class _RemotePageState extends State<RemotePage> {
                       ? []
                       : gFFI.ffiModel.isPeerAndroid
                           ? [
+                              IconButton(
+                                  color: Colors.white,
+                                  icon: Icon(Icons.keyboard),
+                                  onPressed: openKeyboard),
                               IconButton(
                                 color: Colors.white,
                                 icon: const Icon(Icons.build),

libs/scrap/src/android/ffi.rs

@@ -173,6 +173,26 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) ->
     }
 }
 
+pub fn call_main_service_key_event(data: &[u8]) -> JniResult<()> {
+    if let (Some(jvm), Some(ctx)) = (
+        JVM.read().unwrap().as_ref(),
+        MAIN_SERVICE_CTX.read().unwrap().as_ref(),
+    ) {
+        let mut env = jvm.attach_current_thread_as_daemon()?;
+        let data = env.byte_array_from_slice(data)?;
+
+        env.call_method(
+            ctx,
+            "rustKeyEventInput",
+            "([B)V",
+            &[JValue::Object(&JObject::from(data))],
+        )?;
+        return Ok(());
+    } else {
+        return Err(JniError::ThrowFailed(-1));
+    }
+}
+
 pub fn call_main_service_get_by_name(name: &str) -> JniResult<String> {
     if let (Some(jvm), Some(ctx)) = (
         JVM.read().unwrap().as_ref(),

src/client.rs

@@ -1795,14 +1795,14 @@ impl LoginConfigHandler {
             crate::flutter::push_global_event(crate::flutter::APP_TYPE_MAIN, evt);
         }
         if config.keyboard_mode.is_empty() {
-            if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version)) {
+            if is_keyboard_mode_supported(&KeyboardMode::Map, get_version_number(&pi.version), &pi.platform) {
                 config.keyboard_mode = KeyboardMode::Map.to_string();
             } else {
                 config.keyboard_mode = KeyboardMode::Legacy.to_string();
             }
         } else {
             let keyboard_modes =
-                crate::get_supported_keyboard_modes(get_version_number(&pi.version));
+                crate::get_supported_keyboard_modes(get_version_number(&pi.version), &pi.platform);
             let current_mode = &KeyboardMode::from_str(&config.keyboard_mode).unwrap_or_default();
             if !keyboard_modes.contains(current_mode) {
                 config.keyboard_mode = KeyboardMode::Legacy.to_string();

src/common.rs

@@ -1086,18 +1086,24 @@ pub fn make_privacy_mode_msg(state: back_notification::PrivacyModeState) -> Mess
     make_privacy_mode_msg_with_details(state, "".to_owned())
 }
 
-pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64) -> bool {
+pub fn is_keyboard_mode_supported(keyboard_mode: &KeyboardMode, version_number: i64, peer_platform: &str) -> bool {
     match keyboard_mode {
         KeyboardMode::Legacy => true,
-        KeyboardMode::Map => version_number >= hbb_common::get_version_number("1.2.0"),
+        KeyboardMode::Map => {
+            if peer_platform.to_lowercase() == crate::PLATFORM_ANDROID.to_lowercase() {
+                false
+            } else {
+                version_number >= hbb_common::get_version_number("1.2.0")
+            }
+        }
         KeyboardMode::Translate => version_number >= hbb_common::get_version_number("1.2.0"),
         KeyboardMode::Auto => version_number >= hbb_common::get_version_number("1.2.0"),
     }
 }
 
-pub fn get_supported_keyboard_modes(version: i64) -> Vec<KeyboardMode> {
+pub fn get_supported_keyboard_modes(version: i64, peer_platform: &str) -> Vec<KeyboardMode> {
     KeyboardMode::iter()
-        .filter(|&mode| is_keyboard_mode_supported(mode, version))
+        .filter(|&mode| is_keyboard_mode_supported(mode, version, peer_platform))
         .map(|&mode| mode)
         .collect::<Vec<_>>()
 }

src/flutter_ffi.rs

@@ -395,6 +395,7 @@ pub fn session_is_keyboard_mode_supported(session_id: SessionID, mode: String) -
             SyncReturn(is_keyboard_mode_supported(
                 &mode,
                 session.get_peer_version(),
+                &session.peer_platform()
             ))
         } else {
             SyncReturn(false)

src/keyboard.rs

@@ -28,6 +28,8 @@ const OS_LOWER_WINDOWS: &str = "windows";
 const OS_LOWER_LINUX: &str = "linux";
 #[allow(dead_code)]
 const OS_LOWER_MACOS: &str = "macos";
+#[allow(dead_code)]
+const OS_LOWER_ANDROID: &str = "android";
 
 #[cfg(any(target_os = "windows", target_os = "macos"))]
 static KEYBOARD_HOOKED: AtomicBool = AtomicBool::new(false);
@@ -163,6 +165,21 @@ pub mod client {
         }
     }
 
+    pub fn map_key_to_control_key(key: &rdev::Key) -> Option<ControlKey> {
+        match key {
+            Key::Alt => Some(ControlKey::Alt),
+            Key::ShiftLeft => Some(ControlKey::Shift),
+            Key::ControlLeft => Some(ControlKey::Control),
+            Key::MetaLeft => Some(ControlKey::Meta),
+            Key::AltGr => Some(ControlKey::RAlt),
+            Key::ShiftRight => Some(ControlKey::RShift),
+            Key::ControlRight => Some(ControlKey::RControl),
+            Key::MetaRight => Some(ControlKey::RWin),
+            _ => None,
+        }
+
+    }
+
     pub fn event_lock_screen() -> KeyEvent {
         let mut key_event = KeyEvent::new();
         key_event.set_control_key(ControlKey::LockScreen);
@@ -355,7 +372,7 @@ pub fn get_keyboard_mode_enum(keyboard_mode: &str) -> KeyboardMode {
 }
 
 #[inline]
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 pub fn is_modifier(key: &rdev::Key) -> bool {
     matches!(
         key,
@@ -850,12 +867,14 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) ->
                 rdev::win_scancode_to_macos_code(event.position_code)?
             }
         }
+        OS_LOWER_ANDROID => rdev::win_scancode_to_android_key_code(event.position_code)?,
         _ => rdev::win_scancode_to_linux_code(event.position_code)?,
     };
     #[cfg(target_os = "macos")]
     let keycode = match _peer {
         OS_LOWER_WINDOWS => rdev::macos_code_to_win_scancode(event.platform_code as _)?,
         OS_LOWER_MACOS => event.platform_code as _,
+        OS_LOWER_ANDROID => rdev::macos_code_to_android_key_code(event.platform_code as _)?,
         _ => rdev::macos_code_to_linux_code(event.platform_code as _)?,
     };
     #[cfg(target_os = "linux")]
@@ -868,6 +887,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) ->
                 rdev::linux_code_to_macos_code(event.position_code as _)?
             }
         }
+        OS_LOWER_ANDROID => rdev::linux_code_to_android_key_code(event.position_code as _)?,
         _ => event.position_code as _,
     };
     #[cfg(any(target_os = "android", target_os = "ios"))]
@@ -877,7 +897,7 @@ pub fn map_keyboard_mode(_peer: &str, event: &Event, mut key_event: KeyEvent) ->
     Some(key_event)
 }
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 fn try_fill_unicode(_peer: &str, event: &Event, key_event: &KeyEvent, events: &mut Vec<KeyEvent>) {
     match &event.unicode {
         Some(unicode_info) => {
@@ -1046,12 +1066,14 @@ pub fn translate_keyboard_mode(peer: &str, event: &Event, key_event: KeyEvent) -
     events
 }
 
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 pub fn keycode_to_rdev_key(keycode: u32) -> Key {
     #[cfg(target_os = "windows")]
     return rdev::win_key_from_scancode(keycode);
-    #[cfg(target_os = "linux")]
+    #[cfg(any(target_os = "linux"))]
     return rdev::linux_key_from_code(keycode);
+    #[cfg(any(target_os = "android"))]
+    return rdev::android_key_from_code(keycode);
     #[cfg(target_os = "macos")]
     return rdev::macos_key_from_code(keycode.try_into().unwrap_or_default());
 }

src/server/connection.rs

@@ -38,10 +38,12 @@ use hbb_common::{
         sync::mpsc,
         time::{self, Duration, Instant, Interval},
     },
-    tokio_util::codec::{BytesCodec, Framed},
+    tokio_util::codec::{BytesCodec, Framed}, protobuf::EnumOrUnknown,
 };
 #[cfg(any(target_os = "android", target_os = "ios"))]
-use scrap::android::call_main_service_pointer_input;
+use scrap::android::{call_main_service_pointer_input, call_main_service_key_event};
+#[cfg(target_os = "android")]
+use crate::keyboard::client::map_key_to_control_key;
 use serde_derive::Serialize;
 use serde_json::{json, value::Value};
 use sha2::{Digest, Sha256};
@@ -57,7 +59,7 @@ use system_shutdown;
 
 #[cfg(all(windows, feature = "virtual_display_driver"))]
 use crate::virtual_display_manager;
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
+#[cfg(not(any(target_os = "ios")))]
 use std::collections::HashSet;
 pub type Sender = mpsc::UnboundedSender<(Instant, Arc<Message>)>;
 
@@ -215,7 +217,7 @@ pub struct Connection {
     voice_call_request_timestamp: Option<NonZeroI64>,
     audio_input_device_before_voice_call: Option<String>,
     options_in_login: Option<OptionMessage>,
-    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    #[cfg(not(any(target_os = "ios")))]
     pressed_modifiers: HashSet<rdev::Key>,
     #[cfg(all(target_os = "linux", feature = "linux_headless"))]
     #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -354,7 +356,7 @@ impl Connection {
             voice_call_request_timestamp: None,
             audio_input_device_before_voice_call: None,
             options_in_login: None,
-            #[cfg(not(any(target_os = "android", target_os = "ios")))]
+            #[cfg(not(any(target_os = "ios")))]
             pressed_modifiers: Default::default(),
             #[cfg(all(target_os = "linux", feature = "linux_headless"))]
             #[cfg(not(any(feature = "flatpak", feature = "appimage")))]
@@ -1763,8 +1765,59 @@ impl Connection {
                     }
                     self.update_auto_disconnect_timer();
                 }
-                #[cfg(any(target_os = "android", target_os = "ios"))]
+                #[cfg(any(target_os = "ios"))]
                 Some(message::Union::KeyEvent(..)) => {}
+                #[cfg(any(target_os = "android"))]
+                Some(message::Union::KeyEvent(mut me)) => {
+                    let is_press = (me.press || me.down) && !crate::is_modifier(&me);
+
+                    let key = match me.mode.enum_value() {
+                        Ok(KeyboardMode::Map) => {
+                            Some(crate::keyboard::keycode_to_rdev_key(me.chr()))
+                        }
+                        Ok(KeyboardMode::Translate) => {
+                            if let Some(key_event::Union::Chr(code)) = me.union {
+                                Some(crate::keyboard::keycode_to_rdev_key(code & 0x0000FFFF))
+                            } else {
+                                None
+                            }
+                        }
+                        _ => None,
+                    }
+                    .filter(crate::keyboard::is_modifier);
+
+                    if let Some(key) = key {
+                        if is_press {
+                            self.pressed_modifiers.insert(key);
+                        } else {
+                            self.pressed_modifiers.remove(&key);
+                        }
+                    }
+
+                    let mut modifiers = vec![];
+
+                    for key in self.pressed_modifiers.iter() {
+                        if let Some(control_key) = map_key_to_control_key(key) {
+                            modifiers.push(EnumOrUnknown::new(control_key));
+                        }
+                    }
+
+                    me.modifiers = modifiers;
+
+                    let encode_result = me.write_to_bytes();
+
+                    match encode_result {
+                        Ok(data) => {
+                            let result = call_main_service_key_event(&data);
+                            if let Err(e) = result {
+                                log::debug!("call_main_service_key_event fail: {}", e);
+                            }
+                        }
+                        Err(e) => {
+                            log::debug!("encode key event fail: {}", e);
+                        }
+                    }
+                }
                 #[cfg(not(any(target_os = "android", target_os = "ios")))]
                 Some(message::Union::KeyEvent(me)) => {
                     if self.peer_keyboard_enabled() {

src/ui_session_interface.rs

@@ -1,4 +1,4 @@
-use crate::input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL};
+use crate::{input::{MOUSE_BUTTON_LEFT, MOUSE_TYPE_DOWN, MOUSE_TYPE_UP, MOUSE_TYPE_WHEEL}, common::{is_keyboard_mode_supported, get_supported_keyboard_modes}};
 use async_trait::async_trait;
 use bytes::Bytes;
 use rdev::{Event, EventType::*, KeyCode};
@@ -213,18 +213,36 @@ impl<T: InvokeUiSession> Session<T> {
         self.lc.read().unwrap().version.clone()
     }
 
+    pub fn fallback_keyboard_mode(&self) -> String { 
+        let peer_version = self.get_peer_version();
+        let platform = self.peer_platform();
+
+        let supported_modes = get_supported_keyboard_modes(peer_version, &platform);
+        if let Some(mode) = supported_modes.first() {
+            return mode.to_string();
+        } else {
+            if self.get_peer_version() >= get_version_number("1.2.0") {
+                return KeyboardMode::Map.to_string();
+            } else {
+                return KeyboardMode::Legacy.to_string();
+            }
+        }
+    }
+
     pub fn get_keyboard_mode(&self) -> String {
         let mode = self.lc.read().unwrap().keyboard_mode.clone();
-        if ["map", "translate", "legacy"].contains(&(&mode as &str)) {
-            mode
-        } else {
-            if self.get_peer_version() > hbb_common::get_version_number("1.2.0") {
-                "map"
-            } else {
-                "legacy"
+        let keyboard_mode = KeyboardMode::from_str(&mode);
+
+        let peer_version = self.get_peer_version();
+        let platform = self.peer_platform();
+
+        // Saved keyboard mode still exists in this version.
+        if let Ok(mode) = keyboard_mode {
+            if is_keyboard_mode_supported(&mode, peer_version, &platform) {
+                return mode.to_string();
             }
-            .to_string()
         }
+        self.fallback_keyboard_mode()
     }
 
     pub fn save_keyboard_mode(&self, value: String) {