add pointer device event

Signed-off-by: dignow <linlong1265@gmail.com>

flutter/lib/models/input_model.dart

@@ -345,9 +345,20 @@ class InputModel {
 
   // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures
   void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) {
-    final scale = ((e.scale - _lastScale) * 100).toInt();
+    debugPrint(
+        'REMOVE ME =============================== onPointerPanZoomUpdate ${e.scale}');
+    final scale = ((e.scale - _lastScale) * 1000).toInt();
     _lastScale = e.scale;
 
+    if (scale != 0) {
+      bind.sessionSendPointer(
+          sessionId: sessionId,
+          msg: json.encode({
+            'touch': {'scale': scale}
+          }));
+      return;
+    }
+
     final delta = e.panDelta;
     _trackpadLastDelta = delta;
 
@@ -371,7 +382,7 @@ class InputModel {
     if (x != 0 || y != 0) {
       bind.sessionSendMouse(
           sessionId: sessionId,
-          msg: '{"type": "trackpad", "x": "$x", "y": "$y", "scale": "$scale"}');
+          msg: '{"type": "trackpad", "x": "$x", "y": "$y"}');
     }
   }
 
@@ -427,6 +438,12 @@ class InputModel {
   }
 
   void onPointerPanZoomEnd(PointerPanZoomEndEvent e) {
+    bind.sessionSendPointer(
+        sessionId: sessionId,
+        msg: json.encode({
+          'touch': {'scale': 0}
+        }));
+
     waitLastFlingDone();
     _stopFling = false;
 

libs/hbb_common/protos/message.proto

@@ -111,12 +111,31 @@ message LoginResponse {
   }
 }
 
+message TouchScaleUpdate {
+  // The delta scale factor relative to the previous scale.
+  // delta * 1000
+  // 0 means scale end
+  int32 scale = 1;
+}
+
+message TouchEvent {
+  oneof union {
+    TouchScaleUpdate scale_update = 1;
+  }
+  repeated ControlKey modifiers = 2;
+}
+
+message PointerDeviceEvent {
+  oneof union {
+    TouchEvent touch_event = 1;
+  }
+}
+
 message MouseEvent {
   int32 mask = 1;
   sint32 x = 2;
   sint32 y = 3;
   repeated ControlKey modifiers = 4;
-  sint32 scale = 5;
 }
 
 enum KeyboardMode{
@@ -683,5 +702,6 @@ message Message {
     VoiceCallRequest voice_call_request = 23;
     VoiceCallResponse voice_call_response = 24;
     PeerInfo peer_info = 25;
+    PointerDeviceEvent pointer_device_event = 26;
   }
 }

src/client.rs

@@ -1929,7 +1929,6 @@ pub fn send_mouse(
     mask: i32,
     x: i32,
     y: i32,
-    scale: i32,
     alt: bool,
     ctrl: bool,
     shift: bool,
@@ -1941,7 +1940,6 @@ pub fn send_mouse(
         mask,
         x,
         y,
-        scale,
         ..Default::default()
     };
     if alt {
@@ -1968,18 +1966,54 @@ pub fn send_mouse(
     interface.send(Data::Message(msg_out));
 }
 
+#[inline]
+pub fn send_touch(
+    mut evt: TouchEvent,
+    alt: bool,
+    ctrl: bool,
+    shift: bool,
+    command: bool,
+    interface: &impl Interface,
+) {
+    let mut msg_out = Message::new();
+    if alt {
+        evt.modifiers.push(ControlKey::Alt.into());
+    }
+    if shift {
+        evt.modifiers.push(ControlKey::Shift.into());
+    }
+    if ctrl {
+        evt.modifiers.push(ControlKey::Control.into());
+    }
+    if command {
+        evt.modifiers.push(ControlKey::Meta.into());
+    }
+    #[cfg(all(target_os = "macos", not(feature = "flutter")))]
+    if check_scroll_on_mac(mask, x, y) {
+        let factor = 3;
+        mouse_event.mask = crate::input::MOUSE_TYPE_TRACKPAD;
+        mouse_event.x *= factor;
+        mouse_event.y *= factor;
+    }
+    msg_out.set_pointer_device_event(PointerDeviceEvent {
+        union: Some(pointer_device_event::Union::TouchEvent(evt)),
+        ..Default::default()
+    });
+    interface.send(Data::Message(msg_out));
+}
+
 /// Activate OS by sending mouse movement.
 ///
 /// # Arguments
 ///
 /// * `interface` - The interface for sending data.
 fn activate_os(interface: &impl Interface) {
-    send_mouse(0, 0, 0, 0, false, false, false, false, interface);
+    send_mouse(0, 0, 0, false, false, false, false, interface);
     std::thread::sleep(Duration::from_millis(50));
-    send_mouse(0, 3, 3, 0, false, false, false, false, interface);
+    send_mouse(0, 3, 3, false, false, false, false, interface);
     std::thread::sleep(Duration::from_millis(50));
-    send_mouse(1 | 1 << 3, 0, 0, 0, false, false, false, false, interface);
-    send_mouse(2 | 1 << 3, 0, 0, 0, false, false, false, false, interface);
+    send_mouse(1 | 1 << 3, 0, 0, false, false, false, false, interface);
+    send_mouse(2 | 1 << 3, 0, 0, false, false, false, false, interface);
     /*
     let mut key_event = KeyEvent::new();
     // do not use Esc, which has problem with Linux

src/flutter_ffi.rs

@@ -1069,6 +1069,24 @@ pub fn main_start_dbus_server() {
     }
 }
 
+pub fn session_send_pointer(session_id: SessionID, msg: String) {
+    if let Ok(m) = serde_json::from_str::<HashMap<String, serde_json::Value>>(&msg) {
+        let alt = m.get("alt").is_some();
+        let ctrl = m.get("ctrl").is_some();
+        let shift = m.get("shift").is_some();
+        let command = m.get("command").is_some();
+        if let Some(touch_event) = m.get("touch") {
+            if let Some(scale) = touch_event.get("scale") {
+                if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
+                    if let Some(scale) = scale.as_i64() {
+                        session.send_touch_scale(scale as _, alt, ctrl, shift, command);
+                    }
+                }
+            }
+        }
+    }
+}
+
 pub fn session_send_mouse(session_id: SessionID, msg: String) {
     if let Ok(m) = serde_json::from_str::<HashMap<String, String>>(&msg) {
         let alt = m.get("alt").is_some();
@@ -1103,12 +1121,8 @@ pub fn session_send_mouse(session_id: SessionID, msg: String) {
                 _ => 0,
             } << 3;
         }
-        let scale = m
-            .get("scale")
-            .map(|x| x.parse::<i32>().unwrap_or(0))
-            .unwrap_or(0);
         if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
-            session.send_mouse(mask, x, y, scale, alt, ctrl, shift, command);
+            session.send_mouse(mask, x, y, alt, ctrl, shift, command);
         }
     }
 }

src/ipc.rs

@@ -152,6 +152,7 @@ pub enum DataPortableService {
     Pong,
     ConnCount(Option<usize>),
     Mouse((Vec<u8>, i32)),
+    Pointer((Vec<u8>, i32)),
     Key(Vec<u8>),
     RequestStart,
     WillClose,

src/server/connection.rs

@@ -115,6 +115,8 @@ enum MessageInput {
     Mouse((MouseEvent, i32)),
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     Key((KeyEvent, bool)),
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    Pointer((PointerDeviceEvent, i32)),
     BlockOn,
     BlockOff,
     #[cfg(all(feature = "flutter", feature = "plugin_framework"))]
@@ -668,6 +670,9 @@ impl Connection {
                             handle_key(&msg);
                         }
                     }
+                    MessageInput::Pointer((msg, id)) => {
+                        handle_pointer(&msg, id);
+                    }
                     MessageInput::BlockOn => {
                         if crate::platform::block_input(true) {
                             block_input_mode = true;
@@ -1179,6 +1184,12 @@ impl Connection {
         self.tx_input.send(MessageInput::Mouse((msg, conn_id))).ok();
     }
 
+    #[inline]
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+    fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) {
+        self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok();
+    }
+
     #[inline]
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     fn input_key(&self, msg: KeyEvent, press: bool) {
@@ -1577,6 +1588,13 @@ impl Connection {
                         self.input_mouse(me, self.inner.id());
                     }
                 }
+                Some(message::Union::PointerDeviceEvent(pde)) => {
+                    #[cfg(not(any(target_os = "android", target_os = "ios")))]
+                    if self.peer_keyboard_enabled() {
+                        MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);
+                        self.input_pointer(pde, self.inner.id());
+                    }
+                }
                 #[cfg(any(target_os = "android", target_os = "ios"))]
                 Some(message::Union::KeyEvent(..)) => {}
                 #[cfg(not(any(target_os = "android", target_os = "ios")))]

src/server/input_service.rs

@@ -7,7 +7,11 @@ use crate::input::*;
 #[cfg(target_os = "macos")]
 use dispatch::Queue;
 use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable};
-use hbb_common::{get_time, protobuf::EnumOrUnknown};
+use hbb_common::{
+    get_time,
+    message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate},
+    protobuf::EnumOrUnknown,
+};
 use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey};
 #[cfg(target_os = "macos")]
 use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput};
@@ -523,6 +527,21 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) {
     handle_mouse_(evt, conn);
 }
 
+// to-do: merge handle_mouse and handle_pointer
+pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) {
+    #[cfg(target_os = "macos")]
+    if !is_server() {
+        // having GUI, run main GUI thread, otherwise crash
+        let evt = evt.clone();
+        QUEUE.exec_async(move || handle_pointer_(&evt, conn));
+        return;
+    }
+    #[cfg(windows)]
+    crate::portable_service::client::handle_pointer(evt, conn);
+    #[cfg(not(windows))]
+    handle_pointer_(evt, conn);
+}
+
 pub fn fix_key_down_timeout_loop() {
     std::thread::spawn(move || loop {
         std::thread::sleep(std::time::Duration::from_millis(10_000));
@@ -743,7 +762,7 @@ fn active_mouse_(conn: i32) -> bool {
     }
 }
 
-pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
+pub fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) {
     if !active_mouse_(conn) {
         return;
     }
@@ -752,12 +771,25 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
         return;
     }
 
-    if evt.scale != 0 {
+    match &evt.union {
+        Some(TouchEvent(evt)) => match &evt.union {
+            Some(ScaleUpdate(_scale_evt)) => {
                 #[cfg(target_os = "windows")]
-        {
-            handle_scale(evt.scale);
+                handle_scale(_scale_evt.scale);
+            }
+            _ => {}
+        },
+        _ => {}
+    }
+}
+
+pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
+    if !active_mouse_(conn) {
         return;
     }
+
+    if EXITING.load(Ordering::SeqCst) {
+        return;
     }
 
     #[cfg(windows)]
@@ -896,10 +928,13 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
 #[cfg(target_os = "windows")]
 fn handle_scale(scale: i32) {
     let mut en = ENIGO.lock().unwrap();
+    if scale == 0 {
+        en.key_up(Key::Control);
+    } else {
         if en.key_down(Key::Control).is_ok() {
             en.mouse_scroll_y(scale);
         }
-    en.key_up(Key::Control);
+    }
 }
 
 pub fn is_enter(evt: &KeyEvent) -> bool {

src/server/portable_service.rs

@@ -222,6 +222,8 @@ mod utils {
 
 // functions called in separate SYSTEM user process.
 pub mod server {
+    use hbb_common::message_proto::PointerDeviceEvent;
+
     use super::*;
 
     lazy_static::lazy_static! {
@@ -466,6 +468,11 @@ pub mod server {
                                             crate::input_service::handle_mouse_(&evt, conn);
                                         }
                                     }
+                                    Pointer((v, conn)) => {
+                                        if let Ok(evt) = PointerDeviceEvent::parse_from_bytes(&v) {
+                                            crate::input_service::handle_pointer_(&evt, conn);
+                                        }
+                                    }
                                     Key(v) => {
                                         if let Ok(evt) = KeyEvent::parse_from_bytes(&v) {
                                             crate::input_service::handle_key_(&evt);
@@ -499,7 +506,7 @@ pub mod server {
 
 // functions called in main process.
 pub mod client {
-    use hbb_common::anyhow::Context;
+    use hbb_common::{anyhow::Context, message_proto::PointerDeviceEvent};
 
     use super::*;
 
@@ -864,6 +871,14 @@ pub mod client {
         ))))
     }
 
+    fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) -> ResultType<()> {
+        let mut v = vec![];
+        evt.write_to_vec(&mut v)?;
+        ipc_send(Data::DataPortableService(DataPortableService::Pointer((
+            v, conn,
+        ))))
+    }
+
     fn handle_key_(evt: &KeyEvent) -> ResultType<()> {
         let mut v = vec![];
         evt.write_to_vec(&mut v)?;
@@ -910,6 +925,15 @@ pub mod client {
         }
     }
 
+    pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) {
+        if RUNNING.lock().unwrap().clone() {
+            crate::input_service::update_latest_input_cursor_time(conn);
+            handle_pointer_(evt, conn).ok();
+        } else {
+            crate::input_service::handle_pointer_(evt, conn);
+        }
+    }
+
     pub fn handle_key(evt: &KeyEvent) {
         if RUNNING.lock().unwrap().clone() {
             handle_key_(evt).ok();

src/ui_session_interface.rs

@@ -35,8 +35,8 @@ use hbb_common::{
 use crate::client::io_loop::Remote;
 use crate::client::{
     check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay,
-    input_os_password, load_config, send_mouse, start_video_audio_threads, FileManager, Key,
-    LoginConfigHandler, QualityStatus, KEY_MAP,
+    input_os_password, load_config, send_mouse, send_touch, start_video_audio_threads, FileManager,
+    Key, LoginConfigHandler, QualityStatus, KEY_MAP,
 };
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use crate::common::GrabState;
@@ -690,12 +690,23 @@ impl<T: InvokeUiSession> Session<T> {
         self.send_key_event(&key_event);
     }
 
+    pub fn send_touch_scale(&self, scale: i32, alt: bool, ctrl: bool, shift: bool, command: bool) {
+        let scale_evt = TouchScaleUpdate {
+            scale,
+            ..Default::default()
+        };
+        let evt = TouchEvent {
+            union: Some(touch_event::Union::ScaleUpdate(scale_evt)),
+            ..Default::default()
+        };
+        send_touch(evt, alt, ctrl, shift, command, self);
+    }
+
     pub fn send_mouse(
         &self,
         mask: i32,
         x: i32,
         y: i32,
-        scale: i32,
         alt: bool,
         ctrl: bool,
         shift: bool,
@@ -714,7 +725,7 @@ impl<T: InvokeUiSession> Session<T> {
         let (alt, ctrl, shift, command) =
             keyboard::client::get_modifiers_state(alt, ctrl, shift, command);
 
-        send_mouse(mask, x, y, scale, alt, ctrl, shift, command, self);
+        send_mouse(mask, x, y, alt, ctrl, shift, command, self);
         // on macos, ctrl + left button down = right button down, up won't emit, so we need to
         // emit up myself if peer is not macos
         // to-do: how about ctrl + left from win to macos
@@ -730,7 +741,6 @@ impl<T: InvokeUiSession> Session<T> {
                     (MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _,
                     x,
                     y,
-                    scale,
                     alt,
                     ctrl,
                     shift,