feat: support android keyboard input

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -10,8 +10,10 @@ import android.accessibilityservice.AccessibilityService
 import android.accessibilityservice.GestureDescription
 import android.graphics.Path
 import android.os.Build
+import android.os.Bundle
 import android.util.Log
 import android.view.accessibility.AccessibilityEvent
+import android.view.accessibility.AccessibilityNodeInfo
 import androidx.annotation.RequiresApi
 import java.util.*
 import kotlin.math.abs
@@ -252,6 +254,18 @@ class InputService : AccessibilityService() {
         }
     }
 
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun onTextInput(str: String) {
+        findFocus(AccessibilityNodeInfo.FOCUS_INPUT)?.let {
+            val arguments = Bundle()
+            arguments.putCharSequence(
+                AccessibilityNodeInfo.ACTION_ARGUMENT_SET_TEXT_CHARSEQUENCE,
+                str
+            )
+            it.performAction(AccessibilityNodeInfo.ACTION_SET_TEXT, arguments)
+        }
+    }
+
     override fun onServiceConnected() {
         super.onServiceConnected()
         ctx = this

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -94,6 +94,12 @@ class MainService : Service() {
         }
     }
 
+    @Keep
+    @RequiresApi(Build.VERSION_CODES.N)
+    fun rustInputString(input: String) {
+        InputService.ctx?.onTextInput(input)
+    }
+
     @Keep
     fun rustGetByName(name: String): String {
         return when (name) {

flutter/lib/desktop/screen/desktop_remote_screen.dart

@@ -12,9 +12,9 @@ class DesktopRemoteScreen extends StatelessWidget {
   final Map<String, dynamic> params;
 
   DesktopRemoteScreen({Key? key, required this.params}) : super(key: key) {
-    if (!bind.mainStartGrabKeyboard()) {
-      stateGlobal.grabKeyboard = true;
-    }
+      if (!bind.mainStartGrabKeyboard()) {
+        stateGlobal.grabKeyboard = true;
+      }
   }
 
   @override

flutter/lib/mobile/pages/remote_page.dart

@@ -364,6 +364,10 @@ class _RemotePageState extends State<RemotePage> {
                       ? []
                       : gFFI.ffiModel.isPeerAndroid
                           ? [
+                              IconButton(
+                                  color: Colors.white,
+                                  icon: Icon(Icons.keyboard),
+                                  onPressed: openKeyboard),
                               IconButton(
                                 color: Colors.white,
                                 icon: const Icon(Icons.build),

libs/scrap/src/android/ffi.rs

@@ -173,6 +173,25 @@ pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) ->
     }
 }
 
+pub fn call_main_service_input_string(str: &str) -> JniResult<()> {
+    if let (Some(jvm), Some(ctx)) = (
+        JVM.read().unwrap().as_ref(),
+        MAIN_SERVICE_CTX.read().unwrap().as_ref(),
+    ) {
+        let mut env = jvm.attach_current_thread_as_daemon()?;
+        let input_string = env.new_string(str)?;
+        env.call_method(
+            ctx,
+            "rustInputString",
+            "(Ljava/lang/String;)V",
+            &[JValue::Object(&JObject::from(input_string))],
+        )?;
+        return Ok(());
+    } else {
+        return Err(JniError::ThrowFailed(-1));
+    }
+}
+
 pub fn call_main_service_get_by_name(name: &str) -> JniResult<String> {
     if let (Some(jvm), Some(ctx)) = (
         JVM.read().unwrap().as_ref(),

src/server/connection.rs

@@ -41,7 +41,7 @@ use hbb_common::{
     tokio_util::codec::{BytesCodec, Framed},
 };
 #[cfg(any(target_os = "android", target_os = "ios"))]
-use scrap::android::call_main_service_pointer_input;
+use scrap::android::{call_main_service_pointer_input, call_main_service_input_string};
 use serde_json::{json, value::Value};
 use sha2::{Digest, Sha256};
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -1722,8 +1722,17 @@ impl Connection {
                     }
                     self.update_auto_disconnect_timer();
                 }
-                #[cfg(any(target_os = "android", target_os = "ios"))]
+                #[cfg(any(target_os = "ios"))]
                 Some(message::Union::KeyEvent(..)) => {}
+                #[cfg(any(target_os = "android"))]
+                Some(message::Union::KeyEvent(me)) => {
+                    // We can only use seq of key event, android device doesn't support abritrary key stroke.
+                    let seq = me.seq();
+                    let result = call_main_service_input_string(seq);
+                    if let Err(e) = result {
+                        log::debug!("call_main_service_input_string fail:{}", e);
+                    }
+                }
                 #[cfg(not(any(target_os = "android", target_os = "ios")))]
                 Some(message::Union::KeyEvent(me)) => {
                     if self.peer_keyboard_enabled() {