Merge pull request #2100 from fufesou/fix_cursor

Fix cursor

flutter/lib/desktop/pages/remote_page.dart

@@ -251,14 +251,12 @@ class _RemotePageState extends State<RemotePage>
   bool get wantKeepAlive => true;
 }
 
-class ImagePaint extends StatelessWidget {
+class ImagePaint extends StatefulWidget {
   final String id;
   final Rx<bool> cursorOverImage;
   final Rx<bool> keyboardEnabled;
   final Rx<bool> remoteCursorMoved;
   final Widget Function(Widget)? listenerBuilder;
-  final ScrollController _horizontal = ScrollController();
-  final ScrollController _vertical = ScrollController();
 
   ImagePaint(
       {Key? key,
@@ -269,6 +267,21 @@ class ImagePaint extends StatelessWidget {
       this.listenerBuilder})
       : super(key: key);
 
+  @override
+  State<StatefulWidget> createState() => _ImagePaintState();
+}
+
+class _ImagePaintState extends State<ImagePaint> {
+  bool _lastRemoteCursorMoved = false;
+  final ScrollController _horizontal = ScrollController();
+  final ScrollController _vertical = ScrollController();
+
+  String get id => widget.id;
+  Rx<bool> get cursorOverImage => widget.cursorOverImage;
+  Rx<bool> get keyboardEnabled => widget.keyboardEnabled;
+  Rx<bool> get remoteCursorMoved => widget.remoteCursorMoved;
+  Widget Function(Widget)? get listenerBuilder => widget.listenerBuilder;
+
   @override
   Widget build(BuildContext context) {
     final m = Provider.of<ImageModel>(context);
@@ -278,9 +291,18 @@ class ImagePaint extends StatelessWidget {
     mouseRegion({child}) => Obx(() => MouseRegion(
         cursor: cursorOverImage.isTrue
             ? keyboardEnabled.isTrue
-                ? (remoteCursorMoved.isTrue
+                ? (() {
-                    ? SystemMouseCursors.none
+                    if (remoteCursorMoved.isTrue) {
-                    : _buildCustomCursor(context, s))
+                      _lastRemoteCursorMoved = true;
+                      return SystemMouseCursors.none;
+                    } else {
+                      if (_lastRemoteCursorMoved) {
+                        _lastRemoteCursorMoved = false;
+                        _firstEnterImage.value = true;
+                      }
+                      return _buildCustomCursor(context, s);
+                    }
+                  }())
                 : _buildDisabledCursor(context, s)
             : MouseCursor.defer,
         onHover: (evt) {},
@@ -340,10 +362,8 @@ class ImagePaint extends StatelessWidget {
       return FlutterCustomMemoryImageCursor(
         pixbuf: cache.data,
         key: key,
-        // hotx: cache.hotx,
+        hotx: cache.hotx,
-        // hoty: cache.hoty,
+        hoty: cache.hoty,
-        hotx: 0,
-        hoty: 0,
         imageWidth: (cache.width * cache.scale).toInt(),
         imageHeight: (cache.height * cache.scale).toInt(),
       );
@@ -488,11 +508,19 @@ class CursorPaint extends StatelessWidget {
     final m = Provider.of<CursorModel>(context);
     final c = Provider.of<CanvasModel>(context);
     // final adjust = m.adjustForKeyboard();
+    double hotx = m.hotx;
+    double hoty = m.hoty;
+    if (m.image == null) {
+      if (m.defaultCache != null) {
+        hotx = m.defaultImage!.width / 2;
+        hoty = m.defaultImage!.height / 2;
+      }
+    }
     return CustomPaint(
       painter: ImagePainter(
-          image: m.image,
+          image: m.image ?? m.defaultImage,
-          x: m.x - m.hotx + c.x / c.scale,
+          x: m.x - hotx + c.x / c.scale,
-          y: m.y - m.hoty + c.y / c.scale,
+          y: m.y - hoty + c.y / c.scale,
           scale: c.scale),
     );
   }

flutter/lib/mobile/pages/remote_page.dart

@@ -862,11 +862,19 @@ class CursorPaint extends StatelessWidget {
     final c = Provider.of<CanvasModel>(context);
     final adjust = gFFI.cursorModel.adjustForKeyboard();
     var s = c.scale;
+    double hotx = m.hotx;
+    double hoty = m.hoty;
+    if (m.image == null) {
+      if (m.defaultCache != null) {
+        hotx = m.defaultImage!.width / 2;
+        hoty = m.defaultImage!.height / 2;
+      }
+    }
     return CustomPaint(
       painter: ImagePainter(
-          image: m.image,
+          image: m.image ?? m.defaultImage,
-          x: m.x * s - m.hotx + c.x,
+          x: m.x * s - hotx * s + c.x,
-          y: m.y * s - m.hoty + c.y - adjust,
+          y: m.y * s - hoty * s + c.y - adjust,
           scale: 1),
     );
   }

flutter/lib/models/input_model.dart

@@ -42,7 +42,7 @@ class InputModel {
   // mouse
   final isPhysicalMouse = false.obs;
   int _lastMouseDownButtons = 0;
-  Offset last_mouse_pos = Offset.zero;
+  Offset lastMousePos = Offset.zero;
 
   get id => parent.target?.id ?? "";
 
@@ -308,23 +308,23 @@ class InputModel {
     double y = max(0.0, evt['y']);
     final cursorModel = parent.target!.cursorModel;
 
-    if (cursorModel.is_peer_control_protected) {
+    if (cursorModel.isPeerControlProtected) {
-      last_mouse_pos = ui.Offset(x, y);
+      lastMousePos = ui.Offset(x, y);
       return;
     }
 
-    if (!cursorModel.got_mouse_control) {
+    if (!cursorModel.gotMouseControl) {
-      bool self_get_control =
+      bool selfGetControl =
-          (x - last_mouse_pos.dx).abs() > kMouseControlDistance ||
+          (x - lastMousePos.dx).abs() > kMouseControlDistance ||
-              (y - last_mouse_pos.dy).abs() > kMouseControlDistance;
+              (y - lastMousePos.dy).abs() > kMouseControlDistance;
-      if (self_get_control) {
+      if (selfGetControl) {
-        cursorModel.got_mouse_control = true;
+        cursorModel.gotMouseControl = true;
       } else {
-        last_mouse_pos = ui.Offset(x, y);
+        lastMousePos = ui.Offset(x, y);
         return;
       }
     }
-    last_mouse_pos = ui.Offset(x, y);
+    lastMousePos = ui.Offset(x, y);
 
     var type = '';
     var isMove = false;

flutter/lib/models/model.dart

@@ -721,11 +721,14 @@ class CursorData {
               height: (height * scale).toInt(),
             )
             .getBytes(format: img2.Format.bgra);
-        hotx = (width * scale) / 2;
-        hoty = (height * scale) / 2;
       }
     }
     this.scale = scale;
+    if (hotx > 0 && hoty > 0) {
+      // default cursor data
+      hotx = (width * scale) / 2;
+      hoty = (height * scale) / 2;
+    }
     return scale;
   }
 
@@ -737,6 +740,7 @@ class CursorData {
 
 class CursorModel with ChangeNotifier {
   ui.Image? _image;
+  ui.Image? _defaultImage;
   final _images = <int, Tuple3<ui.Image, double, double>>{};
   CursorData? _cache;
   final _defaultCacheId = -1;
@@ -749,13 +753,14 @@ class CursorModel with ChangeNotifier {
   double _hoty = 0;
   double _displayOriginX = 0;
   double _displayOriginY = 0;
-  bool got_mouse_control = true;
+  bool gotMouseControl = true;
-  DateTime _last_peer_mouse = DateTime.now()
+  DateTime _lastPeerMouse = DateTime.now()
       .subtract(Duration(milliseconds: 2 * kMouseControlTimeoutMSec));
   String id = '';
   WeakReference<FFI> parent;
 
   ui.Image? get image => _image;
+  ui.Image? get defaultImage => _defaultImage;
   CursorData? get cache => _cache;
   CursorData? get defaultCache => _getDefaultCache();
 
@@ -767,34 +772,52 @@ class CursorModel with ChangeNotifier {
   double get hotx => _hotx;
   double get hoty => _hoty;
 
-  bool get is_peer_control_protected =>
+  bool get isPeerControlProtected =>
-      DateTime.now().difference(_last_peer_mouse).inMilliseconds <
+      DateTime.now().difference(_lastPeerMouse).inMilliseconds <
       kMouseControlTimeoutMSec;
 
-  CursorModel(this.parent);
+  CursorModel(this.parent) {
+    _getDefaultImage();
+    _getDefaultCache();
+  }
 
   Set<String> get cachedKeys => _cacheKeys;
   addKey(String key) => _cacheKeys.add(key);
 
+  Future<ui.Image?> _getDefaultImage() async {
+    if (_defaultImage == null) {
+      final defaultImg = defaultCursorImage!;
+      // This function is called only one time, no need to care about the performance.
+      Uint8List data = defaultImg.getBytes(format: img2.Format.rgba);
+      _defaultImage = await img.decodeImageFromPixels(
+          data, defaultImg.width, defaultImg.height, ui.PixelFormat.rgba8888);
+    }
+    return _defaultImage;
+  }
+
   CursorData? _getDefaultCache() {
     if (_defaultCache == null) {
+      Uint8List data;
+      double scale = 1.0;
+      double hotx = (defaultCursorImage!.width * scale) / 2;
+      double hoty = (defaultCursorImage!.height * scale) / 2;
       if (Platform.isWindows) {
-        Uint8List data = defaultCursorImage!.getBytes(format: img2.Format.bgra);
+        data = defaultCursorImage!.getBytes(format: img2.Format.bgra);
-        _hotx = defaultCursorImage!.width / 2;
+      } else {
-        _hoty = defaultCursorImage!.height / 2;
+        data = Uint8List.fromList(img2.encodePng(defaultCursorImage!));
-
-        _defaultCache = CursorData(
-          peerId: id,
-          id: _defaultCacheId,
-          image: defaultCursorImage?.clone(),
-          scale: 1.0,
-          data: data,
-          hotx: _hotx,
-          hoty: _hoty,
-          width: defaultCursorImage!.width,
-          height: defaultCursorImage!.height,
-        );
       }
+
+      _defaultCache = CursorData(
+        peerId: id,
+        id: _defaultCacheId,
+        image: defaultCursorImage?.clone(),
+        scale: scale,
+        data: data,
+        hotx: hotx,
+        hoty: hoty,
+        width: defaultCursorImage!.width,
+        height: defaultCursorImage!.height,
+      );
     }
     return _defaultCache;
   }
@@ -926,13 +949,15 @@ class CursorModel with ChangeNotifier {
     var height = int.parse(evt['height']);
     List<dynamic> colors = json.decode(evt['colors']);
     final rgba = Uint8List.fromList(colors.map((s) => s as int).toList());
-    var pid = parent.target?.id;
     final image = await img.decodeImageFromPixels(
         rgba, width, height, ui.PixelFormat.rgba8888);
-    if (parent.target?.id != pid) return;
     _image = image;
-    _images[id] = Tuple3(image, _hotx, _hoty);
+    if (await _updateCache(image, id, width, height)) {
-    await _updateCache(image, id, width, height);
+      _images[id] = Tuple3(image, _hotx, _hoty);
+    } else {
+      _hotx = 0;
+      _hoty = 0;
+    }
     try {
       // my throw exception, because the listener maybe already dispose
       notifyListeners();
@@ -941,44 +966,33 @@ class CursorModel with ChangeNotifier {
     }
   }
 
-  _updateCache(ui.Image image, int id, int w, int h) async {
+  Future<bool> _updateCache(ui.Image image, int id, int w, int h) async {
-    Uint8List? data;
+    ui.ImageByteFormat imgFormat = ui.ImageByteFormat.png;
-    img2.Image? image2;
     if (Platform.isWindows) {
-      ByteData? data2 =
+      imgFormat = ui.ImageByteFormat.rawRgba;
-          await image.toByteData(format: ui.ImageByteFormat.rawRgba);
-      if (data2 != null) {
-        data = data2.buffer.asUint8List();
-        image2 = img2.Image.fromBytes(w, h, data);
-      } else {
-        data = defaultCursorImage?.getBytes(format: img2.Format.bgra);
-        image2 = defaultCursorImage?.clone();
-        _hotx = defaultCursorImage!.width / 2;
-        _hoty = defaultCursorImage!.height / 2;
-      }
-    } else {
-      ByteData? data2 = await image.toByteData(format: ui.ImageByteFormat.png);
-      if (data2 != null) {
-        data = data2.buffer.asUint8List();
-      } else {
-        data = Uint8List.fromList(img2.encodePng(defaultCursorImage!));
-        _hotx = defaultCursorImage!.width / 2;
-        _hoty = defaultCursorImage!.height / 2;
-      }
     }
 
+    ByteData? imgBytes = await image.toByteData(format: imgFormat);
+    if (imgBytes == null) {
+      return false;
+    }
+
+    Uint8List? data = imgBytes.buffer.asUint8List();
     _cache = CursorData(
       peerId: this.id,
       id: id,
-      image: image2,
+      image: Platform.isWindows ? img2.Image.fromBytes(w, h, data) : null,
       scale: 1.0,
       data: data,
-      hotx: _hotx,
+      hotx: 0,
-      hoty: _hoty,
+      hoty: 0,
+      // hotx: _hotx,
+      // hoty: _hoty,
       width: w,
       height: h,
     );
     _cacheMap[id] = _cache!;
+    return true;
   }
 
   updateCursorId(Map<String, dynamic> evt) async {
@@ -998,8 +1012,8 @@ class CursorModel with ChangeNotifier {
 
   /// Update the cursor position.
   updateCursorPosition(Map<String, dynamic> evt, String id) async {
-    got_mouse_control = false;
+    gotMouseControl = false;
-    _last_peer_mouse = DateTime.now();
+    _lastPeerMouse = DateTime.now();
     _x = double.parse(evt['x']);
     _y = double.parse(evt['y']);
     try {

flutter/pubspec.yaml

@@ -72,7 +72,7 @@ dependencies:
     flutter_custom_cursor:
         git:
             url: https://github.com/Kingtous/rustdesk_flutter_custom_cursor
-            ref: ac3c1bf816197863cdcfa42d008962ff644132b0
+            ref: bfb19c84a8244771488bc05cc5f9c9b5e0324cfd
     window_size:
         git:
             url: https://github.com/google/flutter-desktop-embedding.git

src/server/input_service.rs

@@ -126,7 +126,7 @@ pub fn new_pos() -> GenericService {
 }
 
 fn update_last_cursor_pos(x: i32, y: i32) {
-    let mut lock = LATEST_CURSOR_POS.lock().unwrap();
+    let mut lock = LATEST_SYS_CURSOR_POS.lock().unwrap();
     if lock.1 .0 != x || lock.1 .1 != y {
         (lock.0, lock.1) = (Instant::now(), (x, y))
     }
@@ -144,7 +144,7 @@ fn run_pos(sp: GenericService, state: &mut StatePos) -> ResultType<()> {
             });
             let exclude = {
                 let now = get_time();
-                let lock = LATEST_INPUT_CURSOR.lock().unwrap();
+                let lock = LATEST_PEER_INPUT_CURSOR.lock().unwrap();
                 if now - lock.time < 300 {
                     lock.conn
                 } else {
@@ -203,12 +203,13 @@ lazy_static::lazy_static! {
         Arc::new(Mutex::new(Enigo::new()))
     };
     static ref KEYS_DOWN: Arc<Mutex<HashMap<u64, Instant>>> = Default::default();
-    static ref LATEST_INPUT_CURSOR: Arc<Mutex<Input>> = Default::default();
+    static ref LATEST_PEER_INPUT_CURSOR: Arc<Mutex<Input>> = Default::default();
-    static ref LATEST_CURSOR_POS: Arc<Mutex<(Instant, (i32, i32))>> = Arc::new(Mutex::new((Instant::now().sub(MOUSE_MOVE_PROTECTION_TIMEOUT), (0, 0))));
+    static ref LATEST_SYS_CURSOR_POS: Arc<Mutex<(Instant, (i32, i32))>> = Arc::new(Mutex::new((Instant::now().sub(MOUSE_MOVE_PROTECTION_TIMEOUT), (0, 0))));
 }
 static EXITING: AtomicBool = AtomicBool::new(false);
 
 const MOUSE_MOVE_PROTECTION_TIMEOUT: Duration = Duration::from_millis(1_000);
+// Actual diff of (x,y) is (1,1) here. But 5 may be tolerant.
 const MOUSE_ACTIVE_DISTANCE: i32 = 5;
 
 // mac key input must be run in main thread, otherwise crash on >= osx 10.15
@@ -396,24 +397,42 @@ fn fix_modifiers(modifiers: &[EnumOrUnknown<ControlKey>], en: &mut Enigo, ck: i3
 
 fn active_mouse_(conn: i32) -> bool {
     // out of time protection
-    if LATEST_CURSOR_POS.lock().unwrap().0.elapsed() > MOUSE_MOVE_PROTECTION_TIMEOUT {
+    if LATEST_SYS_CURSOR_POS.lock().unwrap().0.elapsed() > MOUSE_MOVE_PROTECTION_TIMEOUT {
         return true;
     }
 
-    let mut last_input = LATEST_INPUT_CURSOR.lock().unwrap();
     // last conn input may be protected
-    if last_input.conn != conn {
+    if LATEST_PEER_INPUT_CURSOR.lock().unwrap().conn != conn {
         return false;
     }
 
-    // check if input is in valid range
+    let in_actived_dist = |a: i32, b: i32| -> bool { (a - b).abs() < MOUSE_ACTIVE_DISTANCE };
+
+    // Check if input is in valid range
     match crate::get_cursor_pos() {
         Some((x, y)) => {
-            let can_active = (last_input.x - x).abs() < MOUSE_ACTIVE_DISTANCE
+            let (last_in_x, last_in_y) = {
-                && (last_input.y - y).abs() < MOUSE_ACTIVE_DISTANCE;
+                let lock = LATEST_PEER_INPUT_CURSOR.lock().unwrap();
+                (lock.x, lock.y)
+            };
+            let mut can_active =
+                in_actived_dist(last_in_x, x) && in_actived_dist(last_in_y, y);
+            // The cursor may not have been moved to last input position if system is busy now.
+            // While this is not a common case, we check it again after some time later.
             if !can_active {
-                last_input.x = -MOUSE_ACTIVE_DISTANCE * 2;
+                // 10 micros may be enough for system to move cursor.
-                last_input.y = -MOUSE_ACTIVE_DISTANCE * 2;
+                // We do not care about the situation which system is too slow(more than 10 micros is required).
+                std::thread::sleep(std::time::Duration::from_micros(10));
+                // Sleep here can also somehow suppress delay accumulation.
+                if let Some((x2, y2)) = crate::get_cursor_pos() {
+                    can_active =
+                        in_actived_dist(last_in_x, x2) && in_actived_dist(last_in_y, y2);
+                }
+            }
+            if !can_active {
+                let mut lock = LATEST_PEER_INPUT_CURSOR.lock().unwrap();
+                lock.x = INVALID_CURSOR_POS / 2;
+                lock.y = INVALID_CURSOR_POS / 2;
             }
             can_active
         }
@@ -434,15 +453,6 @@ fn handle_mouse_(evt: &MouseEvent, conn: i32) {
     crate::platform::windows::try_change_desktop();
     let buttons = evt.mask >> 3;
     let evt_type = evt.mask & 0x7;
-    if evt_type == 0 {
-        let time = get_time();
-        *LATEST_INPUT_CURSOR.lock().unwrap() = Input {
-            time,
-            conn,
-            x: evt.x,
-            y: evt.y,
-        };
-    }
     let mut en = ENIGO.lock().unwrap();
     #[cfg(not(target_os = "macos"))]
     let mut to_release = Vec::new();
@@ -467,6 +477,14 @@ fn handle_mouse_(evt: &MouseEvent, conn: i32) {
     }
     match evt_type {
         0 => {
+            let time = get_time();
+            *LATEST_PEER_INPUT_CURSOR.lock().unwrap() = Input {
+                time,
+                conn,
+                x: evt.x,
+                y: evt.y,
+            };
+
             en.mouse_move_to(evt.x, evt.y);
         }
         1 => match buttons {