From c86a8fff034c7dd3b3ddf4f33d69eaee5c3538b0 Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 19 Jul 2023 01:18:10 +0800 Subject: [PATCH] add pointer device event Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 21 +++++++++- libs/hbb_common/protos/message.proto | 22 ++++++++++- src/client.rs | 46 +++++++++++++++++++--- src/flutter_ffi.rs | 24 +++++++++--- src/ipc.rs | 1 + src/server/connection.rs | 18 +++++++++ src/server/input_service.rs | 57 ++++++++++++++++++++++------ src/server/portable_service.rs | 26 ++++++++++++- src/ui_session_interface.rs | 20 +++++++--- 9 files changed, 204 insertions(+), 31 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 44a39ffbc..8fa4d1caf 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -345,9 +345,20 @@ class InputModel { // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { - final scale = ((e.scale - _lastScale) * 100).toInt(); + debugPrint( + 'REMOVE ME =============================== onPointerPanZoomUpdate ${e.scale}'); + final scale = ((e.scale - _lastScale) * 1000).toInt(); _lastScale = e.scale; + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + return; + } + final delta = e.panDelta; _trackpadLastDelta = delta; @@ -371,7 +382,7 @@ class InputModel { if (x != 0 || y != 0) { bind.sessionSendMouse( sessionId: sessionId, - msg: '{"type": "trackpad", "x": "$x", "y": "$y", "scale": "$scale"}'); + msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); } } @@ -427,6 +438,12 @@ class InputModel { } void onPointerPanZoomEnd(PointerPanZoomEndEvent e) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': 0} + })); + waitLastFlingDone(); _stopFling = false; diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 20eb08ba5..433e186d5 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -111,12 +111,31 @@ message LoginResponse { } } +message TouchScaleUpdate { + // The delta scale factor relative to the previous scale. + // delta * 1000 + // 0 means scale end + int32 scale = 1; +} + +message TouchEvent { + oneof union { + TouchScaleUpdate scale_update = 1; + } + repeated ControlKey modifiers = 2; +} + +message PointerDeviceEvent { + oneof union { + TouchEvent touch_event = 1; + } +} + message MouseEvent { int32 mask = 1; sint32 x = 2; sint32 y = 3; repeated ControlKey modifiers = 4; - sint32 scale = 5; } enum KeyboardMode{ @@ -683,5 +702,6 @@ message Message { VoiceCallRequest voice_call_request = 23; VoiceCallResponse voice_call_response = 24; PeerInfo peer_info = 25; + PointerDeviceEvent pointer_device_event = 26; } } diff --git a/src/client.rs b/src/client.rs index 6937816a4..798bc83f2 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1929,7 +1929,6 @@ pub fn send_mouse( mask: i32, x: i32, y: i32, - scale: i32, alt: bool, ctrl: bool, shift: bool, @@ -1941,7 +1940,6 @@ pub fn send_mouse( mask, x, y, - scale, ..Default::default() }; if alt { @@ -1968,18 +1966,54 @@ pub fn send_mouse( interface.send(Data::Message(msg_out)); } +#[inline] +pub fn send_touch( + mut evt: TouchEvent, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, + interface: &impl Interface, +) { + let mut msg_out = Message::new(); + if alt { + evt.modifiers.push(ControlKey::Alt.into()); + } + if shift { + evt.modifiers.push(ControlKey::Shift.into()); + } + if ctrl { + evt.modifiers.push(ControlKey::Control.into()); + } + if command { + evt.modifiers.push(ControlKey::Meta.into()); + } + #[cfg(all(target_os = "macos", not(feature = "flutter")))] + if check_scroll_on_mac(mask, x, y) { + let factor = 3; + mouse_event.mask = crate::input::MOUSE_TYPE_TRACKPAD; + mouse_event.x *= factor; + mouse_event.y *= factor; + } + msg_out.set_pointer_device_event(PointerDeviceEvent { + union: Some(pointer_device_event::Union::TouchEvent(evt)), + ..Default::default() + }); + interface.send(Data::Message(msg_out)); +} + /// Activate OS by sending mouse movement. /// /// # Arguments /// /// * `interface` - The interface for sending data. fn activate_os(interface: &impl Interface) { - send_mouse(0, 0, 0, 0, false, false, false, false, interface); + send_mouse(0, 0, 0, false, false, false, false, interface); std::thread::sleep(Duration::from_millis(50)); - send_mouse(0, 3, 3, 0, false, false, false, false, interface); + send_mouse(0, 3, 3, false, false, false, false, interface); std::thread::sleep(Duration::from_millis(50)); - send_mouse(1 | 1 << 3, 0, 0, 0, false, false, false, false, interface); - send_mouse(2 | 1 << 3, 0, 0, 0, false, false, false, false, interface); + send_mouse(1 | 1 << 3, 0, 0, false, false, false, false, interface); + send_mouse(2 | 1 << 3, 0, 0, false, false, false, false, interface); /* let mut key_event = KeyEvent::new(); // do not use Esc, which has problem with Linux diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index e4564f144..011c1b098 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1069,6 +1069,24 @@ pub fn main_start_dbus_server() { } } +pub fn session_send_pointer(session_id: SessionID, msg: String) { + if let Ok(m) = serde_json::from_str::>(&msg) { + let alt = m.get("alt").is_some(); + let ctrl = m.get("ctrl").is_some(); + let shift = m.get("shift").is_some(); + let command = m.get("command").is_some(); + if let Some(touch_event) = m.get("touch") { + if let Some(scale) = touch_event.get("scale") { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + if let Some(scale) = scale.as_i64() { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + } + } + } +} + pub fn session_send_mouse(session_id: SessionID, msg: String) { if let Ok(m) = serde_json::from_str::>(&msg) { let alt = m.get("alt").is_some(); @@ -1103,12 +1121,8 @@ pub fn session_send_mouse(session_id: SessionID, msg: String) { _ => 0, } << 3; } - let scale = m - .get("scale") - .map(|x| x.parse::().unwrap_or(0)) - .unwrap_or(0); if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_mouse(mask, x, y, scale, alt, ctrl, shift, command); + session.send_mouse(mask, x, y, alt, ctrl, shift, command); } } } diff --git a/src/ipc.rs b/src/ipc.rs index 526761c8b..9eaade320 100644 --- a/src/ipc.rs +++ b/src/ipc.rs @@ -152,6 +152,7 @@ pub enum DataPortableService { Pong, ConnCount(Option), Mouse((Vec, i32)), + Pointer((Vec, i32)), Key(Vec), RequestStart, WillClose, diff --git a/src/server/connection.rs b/src/server/connection.rs index 4224d670a..c91b813ff 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -115,6 +115,8 @@ enum MessageInput { Mouse((MouseEvent, i32)), #[cfg(not(any(target_os = "android", target_os = "ios")))] Key((KeyEvent, bool)), + #[cfg(not(any(target_os = "android", target_os = "ios")))] + Pointer((PointerDeviceEvent, i32)), BlockOn, BlockOff, #[cfg(all(feature = "flutter", feature = "plugin_framework"))] @@ -668,6 +670,9 @@ impl Connection { handle_key(&msg); } } + MessageInput::Pointer((msg, id)) => { + handle_pointer(&msg, id); + } MessageInput::BlockOn => { if crate::platform::block_input(true) { block_input_mode = true; @@ -1179,6 +1184,12 @@ impl Connection { self.tx_input.send(MessageInput::Mouse((msg, conn_id))).ok(); } + #[inline] + #[cfg(not(any(target_os = "android", target_os = "ios")))] + fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) { + self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok(); + } + #[inline] #[cfg(not(any(target_os = "android", target_os = "ios")))] fn input_key(&self, msg: KeyEvent, press: bool) { @@ -1577,6 +1588,13 @@ impl Connection { self.input_mouse(me, self.inner.id()); } } + Some(message::Union::PointerDeviceEvent(pde)) => { + #[cfg(not(any(target_os = "android", target_os = "ios")))] + if self.peer_keyboard_enabled() { + MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst); + self.input_pointer(pde, self.inner.id()); + } + } #[cfg(any(target_os = "android", target_os = "ios"))] Some(message::Union::KeyEvent(..)) => {} #[cfg(not(any(target_os = "android", target_os = "ios")))] diff --git a/src/server/input_service.rs b/src/server/input_service.rs index 49baeab6b..8562ca3eb 100644 --- a/src/server/input_service.rs +++ b/src/server/input_service.rs @@ -7,7 +7,11 @@ use crate::input::*; #[cfg(target_os = "macos")] use dispatch::Queue; use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable}; -use hbb_common::{get_time, protobuf::EnumOrUnknown}; +use hbb_common::{ + get_time, + message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate}, + protobuf::EnumOrUnknown, +}; use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey}; #[cfg(target_os = "macos")] use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput}; @@ -523,6 +527,21 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) { handle_mouse_(evt, conn); } +// to-do: merge handle_mouse and handle_pointer +pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) { + #[cfg(target_os = "macos")] + if !is_server() { + // having GUI, run main GUI thread, otherwise crash + let evt = evt.clone(); + QUEUE.exec_async(move || handle_pointer_(&evt, conn)); + return; + } + #[cfg(windows)] + crate::portable_service::client::handle_pointer(evt, conn); + #[cfg(not(windows))] + handle_pointer_(evt, conn); +} + pub fn fix_key_down_timeout_loop() { std::thread::spawn(move || loop { std::thread::sleep(std::time::Duration::from_millis(10_000)); @@ -743,7 +762,7 @@ fn active_mouse_(conn: i32) -> bool { } } -pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { +pub fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) { if !active_mouse_(conn) { return; } @@ -752,12 +771,25 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { return; } - if evt.scale != 0 { - #[cfg(target_os = "windows")] - { - handle_scale(evt.scale); - return; - } + match &evt.union { + Some(TouchEvent(evt)) => match &evt.union { + Some(ScaleUpdate(_scale_evt)) => { + #[cfg(target_os = "windows")] + handle_scale(_scale_evt.scale); + } + _ => {} + }, + _ => {} + } +} + +pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { + if !active_mouse_(conn) { + return; + } + + if EXITING.load(Ordering::SeqCst) { + return; } #[cfg(windows)] @@ -896,10 +928,13 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) { #[cfg(target_os = "windows")] fn handle_scale(scale: i32) { let mut en = ENIGO.lock().unwrap(); - if en.key_down(Key::Control).is_ok() { - en.mouse_scroll_y(scale); + if scale == 0 { + en.key_up(Key::Control); + } else { + if en.key_down(Key::Control).is_ok() { + en.mouse_scroll_y(scale); + } } - en.key_up(Key::Control); } pub fn is_enter(evt: &KeyEvent) -> bool { diff --git a/src/server/portable_service.rs b/src/server/portable_service.rs index b794e9823..c8185b709 100644 --- a/src/server/portable_service.rs +++ b/src/server/portable_service.rs @@ -222,6 +222,8 @@ mod utils { // functions called in separate SYSTEM user process. pub mod server { + use hbb_common::message_proto::PointerDeviceEvent; + use super::*; lazy_static::lazy_static! { @@ -466,6 +468,11 @@ pub mod server { crate::input_service::handle_mouse_(&evt, conn); } } + Pointer((v, conn)) => { + if let Ok(evt) = PointerDeviceEvent::parse_from_bytes(&v) { + crate::input_service::handle_pointer_(&evt, conn); + } + } Key(v) => { if let Ok(evt) = KeyEvent::parse_from_bytes(&v) { crate::input_service::handle_key_(&evt); @@ -499,7 +506,7 @@ pub mod server { // functions called in main process. pub mod client { - use hbb_common::anyhow::Context; + use hbb_common::{anyhow::Context, message_proto::PointerDeviceEvent}; use super::*; @@ -864,6 +871,14 @@ pub mod client { )))) } + fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) -> ResultType<()> { + let mut v = vec![]; + evt.write_to_vec(&mut v)?; + ipc_send(Data::DataPortableService(DataPortableService::Pointer(( + v, conn, + )))) + } + fn handle_key_(evt: &KeyEvent) -> ResultType<()> { let mut v = vec![]; evt.write_to_vec(&mut v)?; @@ -910,6 +925,15 @@ pub mod client { } } + pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) { + if RUNNING.lock().unwrap().clone() { + crate::input_service::update_latest_input_cursor_time(conn); + handle_pointer_(evt, conn).ok(); + } else { + crate::input_service::handle_pointer_(evt, conn); + } + } + pub fn handle_key(evt: &KeyEvent) { if RUNNING.lock().unwrap().clone() { handle_key_(evt).ok(); diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 773eee374..033033faf 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -35,8 +35,8 @@ use hbb_common::{ use crate::client::io_loop::Remote; use crate::client::{ check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay, - input_os_password, load_config, send_mouse, start_video_audio_threads, FileManager, Key, - LoginConfigHandler, QualityStatus, KEY_MAP, + input_os_password, load_config, send_mouse, send_touch, start_video_audio_threads, FileManager, + Key, LoginConfigHandler, QualityStatus, KEY_MAP, }; #[cfg(not(any(target_os = "android", target_os = "ios")))] use crate::common::GrabState; @@ -690,12 +690,23 @@ impl Session { self.send_key_event(&key_event); } + pub fn send_touch_scale(&self, scale: i32, alt: bool, ctrl: bool, shift: bool, command: bool) { + let scale_evt = TouchScaleUpdate { + scale, + ..Default::default() + }; + let evt = TouchEvent { + union: Some(touch_event::Union::ScaleUpdate(scale_evt)), + ..Default::default() + }; + send_touch(evt, alt, ctrl, shift, command, self); + } + pub fn send_mouse( &self, mask: i32, x: i32, y: i32, - scale: i32, alt: bool, ctrl: bool, shift: bool, @@ -714,7 +725,7 @@ impl Session { let (alt, ctrl, shift, command) = keyboard::client::get_modifiers_state(alt, ctrl, shift, command); - send_mouse(mask, x, y, scale, alt, ctrl, shift, command, self); + send_mouse(mask, x, y, alt, ctrl, shift, command, self); // on macos, ctrl + left button down = right button down, up won't emit, so we need to // emit up myself if peer is not macos // to-do: how about ctrl + left from win to macos @@ -730,7 +741,6 @@ impl Session { (MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _, x, y, - scale, alt, ctrl, shift,