fix: android input, soft keyboard, mouse mode (#9797)

Cursor movement in the remote screen.

Signed-off-by: fufesou <linlong1266@gmail.com>

flutter/lib/mobile/pages/remote_page.dart

@@ -1,4 +1,5 @@
 import 'dart:async';
+import 'dart:ui' as ui;
 
 import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
@@ -43,6 +44,7 @@ class _RemotePageState extends State<RemotePage> with WidgetsBindingObserver {
   bool _showGestureHelp = false;
   String _value = '';
   Orientation? _currentOrientation;
+  double _viewInsetsBottom = 0;
 
   Timer? _timerDidChangeMetrics;
 
@@ -132,9 +134,15 @@ class _RemotePageState extends State<RemotePage> with WidgetsBindingObserver {
 
   @override
   void didChangeMetrics() {
+    final newBottom = MediaQueryData.fromView(ui.window).viewInsets.bottom;
     _timerDidChangeMetrics?.cancel();
-    _timerDidChangeMetrics = Timer(Duration(milliseconds: 100), () {
+    _timerDidChangeMetrics = Timer(Duration(milliseconds: 100), () async {
-      gFFI.canvasModel.updateViewStyle(refreshMousePos: false);
+      // We need this comparation because poping up the floating action will also trigger `didChangeMetrics()`.
+      if (newBottom != _viewInsetsBottom) {
+        await gFFI.canvasModel.updateViewStyle(refreshMousePos: false);
+        gFFI.canvasModel.moveToCenterCursor();
+        _viewInsetsBottom = newBottom;
+      }
     });
   }
 
@@ -962,8 +970,10 @@ class ImagePaint extends StatelessWidget {
     final m = Provider.of<ImageModel>(context);
     final c = Provider.of<CanvasModel>(context);
     var s = c.scale;
+    final adjust = c.getAdjustY();
     return CustomPaint(
-      painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s),
+      painter: ImagePainter(
+          image: m.image, x: c.x / s, y: (c.y + adjust) / s, scale: s),
     );
   }
 }
@@ -1008,11 +1018,12 @@ class CursorPaint extends StatelessWidget {
       factor = s / mins;
     }
     final s2 = s < mins ? mins : s;
+    final adjust = c.getAdjustY();
     return CustomPaint(
       painter: ImagePainter(
           image: image,
           x: (m.x - hotx) * factor + c.x / s2,
-          y: (m.y - hoty) * factor + c.y / s2,
+          y: (m.y - hoty) * factor + (c.y + adjust) / s2,
           scale: s2),
     );
   }

flutter/lib/models/model.dart

@@ -1286,20 +1286,18 @@ class ImageModel with ChangeNotifier {
   }
 
   // mobile only
-  // for desktop, height should minus tabbar height
   double get maxScale {
     if (_image == null) return 1.5;
-    final size = MediaQueryData.fromWindow(ui.window).size;
+    final size = parent.target!.canvasModel.getSize();
     final xscale = size.width / _image!.width;
     final yscale = size.height / _image!.height;
     return max(1.5, max(xscale, yscale));
   }
 
   // mobile only
-  // for desktop, height should minus tabbar height
   double get minScale {
     if (_image == null) return 1.5;
-    final size = MediaQueryData.fromWindow(ui.window).size;
+    final size = parent.target!.canvasModel.getSize();
     final xscale = size.width / _image!.width;
     final yscale = size.height / _image!.height;
     return min(xscale, yscale) / 1.5;
@@ -1464,19 +1462,27 @@ class CanvasModel with ChangeNotifier {
   static double get bottomToEdge =>
       isDesktop ? windowBorderWidth + kDragToResizeAreaPadding.bottom : 0;
 
-  updateViewStyle({refreshMousePos = true}) async {
+  Size getSize() {
-    Size getSize() {
+    final mediaData = MediaQueryData.fromView(ui.window);
-      final mediaData = MediaQueryData.fromView(ui.window);
+    final size = mediaData.size;
-      final size = mediaData.size;
+    // If minimized, w or h may be negative here.
-      // If minimized, w or h may be negative here.
+    double w = size.width - leftToEdge - rightToEdge;
-      double w = size.width - leftToEdge - rightToEdge;
+    double h = size.height - topToEdge - bottomToEdge;
-      double h = size.height - topToEdge - bottomToEdge;
+    if (isMobile) {
-      if (isMobile) {
+      h = h -
-        h -= (mediaData.padding.top + mediaData.viewInsets.bottom);
+          mediaData.viewInsets.bottom -
-      }
+          (parent.target?.cursorModel.keyHelpToolsRect?.bottom ?? 0);
-      return Size(w < 0 ? 0 : w, h < 0 ? 0 : h);
     }
+    return Size(w < 0 ? 0 : w, h < 0 ? 0 : h);
+  }
 
+  // mobile only
+  double getAdjustY() {
+    final bottom = parent.target?.cursorModel.keyHelpToolsRect?.bottom ?? 0;
+    return max(bottom - MediaQueryData.fromView(ui.window).padding.top, 0);
+  }
+
+  updateViewStyle({refreshMousePos = true}) async {
     final style = await bind.sessionGetViewStyle(sessionId: sessionId);
     if (style == null) {
       return;
@@ -1636,6 +1642,7 @@ class CanvasModel with ChangeNotifier {
     notifyListeners();
   }
 
+  // mobile only
   updateScale(double v, Offset focalPoint) {
     if (parent.target?.imageModel.image == null) return;
     final s = _scale;
@@ -1647,9 +1654,10 @@ class CanvasModel with ChangeNotifier {
     // (focalPoint.dx - _x_1) / s1 + displayOriginX = (focalPoint.dx - _x_2) / s2 + displayOriginX
     // _x_2 = focalPoint.dx - (focalPoint.dx - _x_1) / s1 * s2
     _x = focalPoint.dx - (focalPoint.dx - _x) / s * _scale;
-    // (focalPoint.dy - _y_1) / s1 + displayOriginY = (focalPoint.dy - _y_2) / s2 + displayOriginY
+    final adjust = getAdjustY();
-    // _y_2 = focalPoint.dy - (focalPoint.dy - _y_1) / s1 * s2
+    // (focalPoint.dy - _y_1 - adjust) / s1 + displayOriginY = (focalPoint.dy - _y_2 - adjust) / s2 + displayOriginY
-    _y = focalPoint.dy - (focalPoint.dy - _y) / s * _scale;
+    // _y_2 = focalPoint.dy - adjust - (focalPoint.dy - _y_1 - adjust) / s1 * s2
+    _y = focalPoint.dy - adjust - (focalPoint.dy - _y - adjust) / s * _scale;
     notifyListeners();
   }
 
@@ -1690,6 +1698,34 @@ class CanvasModel with ChangeNotifier {
         : 0.0;
     setScrollPercent(percentX, percentY);
   }
+
+  // mobile only
+  // Move the canvas to make the cursor visible(center) on the screen.
+  void moveToCenterCursor() {
+    Rect? imageRect = parent.target?.ffiModel.rect;
+    if (imageRect == null) {
+      // unreachable
+      return;
+    }
+    final maxX = 0.0;
+    final minX = _size.width + (imageRect.left - imageRect.right) * _scale;
+    final maxY = 0.0;
+    final minY = _size.height + (imageRect.top - imageRect.bottom) * _scale;
+    Offset offsetToCenter =
+        parent.target?.cursorModel.getCanvasOffsetToCenterCursor() ??
+            Offset.zero;
+    if (minX < 0) {
+      _x = min(max(offsetToCenter.dx, minX), maxX);
+    } else {
+      // _size.width > (imageRect.right, imageRect.left) * _scale, we should not change _x
+    }
+    if (minY < 0) {
+      _y = min(max(offsetToCenter.dy, minY), maxY);
+    } else {
+      // _size.height > (imageRect.bottom - imageRect.top) * _scale, , we should not change _y
+    }
+    notifyListeners();
+  }
 }
 
 // data for cursor
@@ -1884,6 +1920,7 @@ class CursorModel with ChangeNotifier {
   // Because onDoubleTap() doesn't have the `event` parameter, we can't get the touch event's position.
   bool _lastIsBlocked = false;
 
+  Rect? get keyHelpToolsRect => _keyHelpToolsRect;
   keyHelpToolsVisibilityChanged(Rect? r) {
     _keyHelpToolsRect = r;
     if (r == null) {
@@ -1894,6 +1931,15 @@ class CursorModel with ChangeNotifier {
       // `lastIsBlocked` will be set when the cursor is moving or touch somewhere else.
       _lastIsBlocked = true;
     }
+    if (isMobile) {
+      if (r != null || _lastIsBlocked) {
+        () async {
+          await parent.target?.canvasModel
+              .updateViewStyle(refreshMousePos: false);
+          parent.target?.canvasModel.moveToCenterCursor();
+        }();
+      }
+    }
   }
 
   get lastIsBlocked => _lastIsBlocked;
@@ -1932,8 +1978,10 @@ class CursorModel with ChangeNotifier {
   addKey(String key) => _cacheKeys.add(key);
 
   // remote physical display coordinate
+  // For update pan (mobile), onOneFingerPanStart, onOneFingerPanUpdate, onHoldDragUpdate
   Rect getVisibleRect() {
-    final size = MediaQueryData.fromWindow(ui.window).size;
+    final size = parent.target?.canvasModel.getSize() ??
+        MediaQueryData.fromView(ui.window).size;
     final xoffset = parent.target?.canvasModel.x ?? 0;
     final yoffset = parent.target?.canvasModel.y ?? 0;
     final scale = parent.target?.canvasModel.scale ?? 1;
@@ -1942,7 +1990,20 @@ class CursorModel with ChangeNotifier {
     return Rect.fromLTWH(x0, y0, size.width / scale, size.height / scale);
   }
 
-  get keyboardHeight => MediaQueryData.fromWindow(ui.window).viewInsets.bottom;
+  Offset getCanvasOffsetToCenterCursor() {
+    // cursor should be in the center of the visible rect
+    // _x = rect.left + rect.width / 2
+    // _y = rect.right + rect.height / 2
+    // See `getVisibleRect()`
+    // _x = _displayOriginX - xoffset / scale + size.width / scale * 0.5;
+    // _y = _displayOriginY - yoffset / scale + size.height / scale * 0.5;
+    final size = parent.target?.canvasModel.getSize() ??
+        MediaQueryData.fromView(ui.window).size;
+    final xoffset = (_displayOriginX - _x) * scale + size.width * 0.5;
+    final yoffset = (_displayOriginY - _y) * scale + size.height * 0.5;
+    return Offset(xoffset, yoffset);
+  }
+
   get scale => parent.target?.canvasModel.scale ?? 1.0;
 
   // mobile Soft keyboard, block touch event from the KeyHelpTools
@@ -1965,16 +2026,16 @@ class CursorModel with ChangeNotifier {
       return false;
     }
     _lastIsBlocked = false;
-    moveLocal(x, y);
+    moveLocal(x, y, adjust: parent.target?.canvasModel.getAdjustY() ?? 0);
     parent.target?.inputModel.moveMouse(_x, _y);
     return true;
   }
 
-  moveLocal(double x, double y) {
+  moveLocal(double x, double y, {double adjust = 0}) {
     final xoffset = parent.target?.canvasModel.x ?? 0;
     final yoffset = parent.target?.canvasModel.y ?? 0;
     _x = (x - xoffset) / scale + _displayOriginX;
-    _y = (y - yoffset) / scale + _displayOriginY;
+    _y = (y - yoffset - adjust) / scale + _displayOriginY;
     notifyListeners();
   }