android input add wheel;fix UI/service sync

android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt

@@ -9,6 +9,20 @@ import android.util.Log
 import android.view.accessibility.AccessibilityEvent
 import androidx.annotation.Keep
 import androidx.annotation.RequiresApi
+import java.util.*
+
+const val LIFT_DOWN = 9
+const val LIFT_MOVE = 8
+const val LIFT_UP = 10
+const val RIGHT_UP = 18
+const val WHEEL_BUTTON_DOWN = 33
+const val WHEEL_BUTTON_UP = 34
+const val WHEEL_DOWN = 523331
+const val WHEEL_UP = 963
+
+const val WHEEL_STEP = 120
+const val WHEEL_DURATION = 50L
+const val LONG_TAP_DELAY = 200L
 
 class InputService : AccessibilityService() {
 
@@ -26,10 +40,15 @@ class InputService : AccessibilityService() {
 
     private val logTag = "input service"
     private var leftIsDown = false
-    private var mPath = Path()
+    private var touchPath = Path()
-    private var mLastGestureStartTime = 0L
+    private var lastTouchGestureStartTime = 0L
     private var mouseX = 0
     private var mouseY = 0
+    private var timer = Timer()
+    private var recentActionTask: TimerTask? = null
+
+    private val wheelActionsQueue = LinkedList<GestureDescription>()
+    private var isWheelActionsPolling = false
 
     @Keep
     @RequiresApi(Build.VERSION_CODES.N)
@@ -46,13 +65,13 @@ class InputService : AccessibilityService() {
             _y
         }
 
-        if (!(mask == 9 || mask == 10)) {
+        if (mask == 0 || mask == LIFT_MOVE) {
             mouseX = x * SCREEN_INFO.scale
             mouseY = y * SCREEN_INFO.scale
         }
 
         // left button down ,was up
-        if (mask == 9) {
+        if (mask == LIFT_DOWN) {
             leftIsDown = true
             startGesture(mouseX, mouseY)
             return
@@ -64,43 +83,118 @@ class InputService : AccessibilityService() {
         }
 
         // left up ,was down
-        if (mask == 10) {
+        if (mask == LIFT_UP) {
             leftIsDown = false
             endGesture(mouseX, mouseY)
             return
         }
 
-        if (mask == 18) {
+        if (mask == RIGHT_UP) {
             performGlobalAction(GLOBAL_ACTION_BACK)
             return
         }
 
-        if (mask == 34) {
+        // long WHEEL_BUTTON_DOWN -> GLOBAL_ACTION_RECENTS
-            performGlobalAction(GLOBAL_ACTION_HOME)
+        if (mask == WHEEL_BUTTON_DOWN) {
+            timer.purge()
+            recentActionTask = object : TimerTask() {
+                override fun run() {
+                    performGlobalAction(GLOBAL_ACTION_RECENTS)
+                    recentActionTask = null
+                }
+            }
+            timer.schedule(recentActionTask, LONG_TAP_DELAY)
+        }
+
+        // wheel button up
+        if (mask == WHEEL_BUTTON_UP) {
+            if (recentActionTask != null) {
+                recentActionTask!!.cancel()
+                performGlobalAction(GLOBAL_ACTION_HOME)
+            }
+            return
+        }
+
+        if (mask == WHEEL_DOWN) {
+            if (mouseY < WHEEL_STEP) {
+                return
+            }
+            val path = Path()
+            path.moveTo(mouseX.toFloat(), mouseY.toFloat())
+            path.lineTo(mouseX.toFloat(), (mouseY - WHEEL_STEP).toFloat())
+            val stroke = GestureDescription.StrokeDescription(
+                path,
+                0,
+                WHEEL_DURATION
+            )
+            val builder = GestureDescription.Builder()
+            builder.addStroke(stroke)
+            wheelActionsQueue.offer(builder.build())
+            consumeWheelActions()
+
+        }
+
+        if (mask == WHEEL_UP) {
+            if (mouseY < WHEEL_STEP) {
+                return
+            }
+            val path = Path()
+            path.moveTo(mouseX.toFloat(), mouseY.toFloat())
+            path.lineTo(mouseX.toFloat(), (mouseY + WHEEL_STEP).toFloat())
+            val stroke = GestureDescription.StrokeDescription(
+                path,
+                0,
+                WHEEL_DURATION
+            )
+            val builder = GestureDescription.Builder()
+            builder.addStroke(stroke)
+            wheelActionsQueue.offer(builder.build())
+            consumeWheelActions()
+        }
+    }
+
+    @RequiresApi(Build.VERSION_CODES.N)
+    private fun consumeWheelActions() {
+        if (isWheelActionsPolling) {
+            return
+        } else {
+            isWheelActionsPolling = true
+        }
+        wheelActionsQueue.poll()?.let {
+            dispatchGesture(it, null, null)
+            timer.purge()
+            timer.schedule(object : TimerTask() {
+                override fun run() {
+                    isWheelActionsPolling = false
+                    consumeWheelActions()
+                }
+            }, WHEEL_DURATION + 10)
+        } ?: let {
+            isWheelActionsPolling = false
             return
         }
     }
 
     private fun startGesture(x: Int, y: Int) {
-        mPath = Path()
+        touchPath = Path()
-        mPath.moveTo(x.toFloat(), y.toFloat())
+        touchPath.moveTo(x.toFloat(), y.toFloat())
-        mLastGestureStartTime = System.currentTimeMillis()
+        lastTouchGestureStartTime = System.currentTimeMillis()
     }
 
     private fun continueGesture(x: Int, y: Int) {
-        mPath.lineTo(x.toFloat(), y.toFloat())
+        touchPath.lineTo(x.toFloat(), y.toFloat())
     }
 
     @RequiresApi(Build.VERSION_CODES.N)
     private fun endGesture(x: Int, y: Int) {
         try {
-            mPath.lineTo(x.toFloat(), y.toFloat())
+            touchPath.lineTo(x.toFloat(), y.toFloat())
-            var duration = System.currentTimeMillis() - mLastGestureStartTime
+            var duration = System.currentTimeMillis() - lastTouchGestureStartTime
             if (duration <= 0) {
                 duration = 1
             }
             val stroke = GestureDescription.StrokeDescription(
-                mPath,
+                touchPath,
                 0,
                 duration
             )

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -30,6 +30,11 @@ class MainActivity : FlutterActivity() {
     @RequiresApi(Build.VERSION_CODES.M)
     override fun configureFlutterEngine(flutterEngine: FlutterEngine) {
         super.configureFlutterEngine(flutterEngine)
+        if (MainService.isReady) {
+            Intent(activity, MainService::class.java).also {
+                bindService(it, serviceConnection, Context.BIND_AUTO_CREATE)
+            }
+        }
         flutterMethodChannel = MethodChannel(
             flutterEngine.dartExecutor.binaryMessenger,
             channelTag
@@ -41,7 +46,7 @@ class MainActivity : FlutterActivity() {
                         Intent(activity, MainService::class.java).also {
                             bindService(it, serviceConnection, Context.BIND_AUTO_CREATE)
                         }
-                        if (mainService?.isReady == true) {
+                        if (MainService.isReady) {
                             result.success(false)
                             return@setMethodCallHandler
                         }
@@ -93,7 +98,7 @@ class MainActivity : FlutterActivity() {
                         )
                         flutterMethodChannel.invokeMethod(
                             "on_state_changed",
-                            mapOf("name" to "media", "value" to mainService?.isReady.toString())
+                            mapOf("name" to "media", "value" to MainService.isReady.toString())
                         )
                         result.success(true)
                     }

android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -47,8 +47,6 @@ const val DEFAULT_NOTIFY_TEXT = "Service is running"
 const val DEFAULT_NOTIFY_ID = 1
 const val NOTIFY_ID_OFFSET = 100
 
-const val NOTIFY_TYPE_START_CAPTURE = "NOTIFY_TYPE_START_CAPTURE"
-
 const val MIME_TYPE = MediaFormat.MIMETYPE_VIDEO_VP9
 
 // video const
@@ -141,15 +139,19 @@ class MainService : Service() {
         return translateLocale(LOCAL_NAME, input)
     }
 
+    companion object {
+        private var _isReady = false
+        private var _isStart = false
+        val isReady: Boolean
+            get() = _isReady
+        val isStart: Boolean
+            get() = _isStart
+    }
+
     private val logTag = "LOG_SERVICE"
     private val useVP9 = false
     private val binder = LocalBinder()
-    private var _isReady = false
+
-    private var _isStart = false
-    val isReady: Boolean
-        get() = _isReady
-    val isStart: Boolean
-        get() = _isStart
 
     // video
     private var mediaProjection: MediaProjection? = null
@@ -177,6 +179,15 @@ class MainService : Service() {
         startServer()
     }
 
+    override fun onDestroy() {
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
+            InputService.ctx?.disableSelf()
+        }
+        InputService.ctx = null
+        checkMediaPermission()
+        super.onDestroy()
+    }
+
     private fun updateScreenInfo() {
         var w: Int
         var h: Int
@@ -210,7 +221,6 @@ class MainService : Service() {
                     refreshScreen()
                     startCapture()
                 }
-
             }
 
         }
@@ -242,10 +252,7 @@ class MainService : Service() {
                 checkMediaPermission()
                 init(this)
                 _isReady = true
-            } ?: let {
             }
-//        } else if (intent?.action == ACTION_LOGIN_REQ_NOTIFY) {
-//            val notifyLoginRes = intent.getBooleanExtra(EXTRA_LOGIN_REQ_NOTIFY, false)
         }
         return super.onStartCommand(intent, flags, startId)
     }
@@ -345,7 +352,10 @@ class MainService : Service() {
 
         mediaProjection = null
         checkMediaPermission()
-        stopService(Intent(this, InputService::class.java)) // close input service maybe not work
+        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.N) {
+            InputService.ctx?.disableSelf()
+        }
+        InputService.ctx = null
         stopForeground(true)
         stopSelf()
     }
@@ -357,6 +367,12 @@ class MainService : Service() {
                 mapOf("name" to "media", "value" to isReady.toString())
             )
         }
+        Handler(Looper.getMainLooper()).post {
+            MainActivity.flutterMethodChannel.invokeMethod(
+                "on_state_changed",
+                mapOf("name" to "input", "value" to InputService.isOpen.toString())
+            )
+        }
         return isReady
     }
 

lib/common.dart

@@ -248,6 +248,8 @@ class AccessibilityListener extends StatelessWidget {
                 pointer: evt.pointer + offset,
                 size: 0.1,
                 position: evt.position));
+            GestureBinding.instance!.handlePointerEvent(PointerRemovedEvent(
+                pointer: evt.pointer + offset, position: evt.position));
           }
         },
         onPointerMove: (evt) {

lib/models/server_model.dart

@@ -182,6 +182,7 @@ class ServerModel with ChangeNotifier {
     await FFI.invokeMethod("init_service");
     FFI.setByName("start_service");
     getIDPasswd();
+    updateClientState();
   }
 
   Future<Null> stopService() async {
@@ -281,12 +282,13 @@ class ServerModel with ChangeNotifier {
     try {
       final List clientsJson = jsonDecode(res);
       for (var clientJson in clientsJson) {
-        final client = Client.fromJson(jsonDecode(clientJson));
+        final client = Client.fromJson(clientJson);
         _clients[client.id] = client;
       }
-
       notifyListeners();
-    } catch (e) {}
+    } catch (e) {
+      debugPrint("Failed to updateClientState:$e");
+    }
   }
 
   loginRequest(Map<String, dynamic> evt) {