diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart
index 44a39ffbc..8fa4d1caf 100644
--- a/flutter/lib/models/input_model.dart
+++ b/flutter/lib/models/input_model.dart
@@ -345,9 +345,20 @@ class InputModel {
// https://docs.flutter.dev/release/breaking-changes/trackpad-gestures
void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) {
- final scale = ((e.scale - _lastScale) * 100).toInt();
+ debugPrint(
+ 'REMOVE ME =============================== onPointerPanZoomUpdate ${e.scale}');
+ final scale = ((e.scale - _lastScale) * 1000).toInt();
_lastScale = e.scale;
+ if (scale != 0) {
+ bind.sessionSendPointer(
+ sessionId: sessionId,
+ msg: json.encode({
+ 'touch': {'scale': scale}
+ }));
+ return;
+ }
+
final delta = e.panDelta;
_trackpadLastDelta = delta;
@@ -371,7 +382,7 @@ class InputModel {
if (x != 0 || y != 0) {
bind.sessionSendMouse(
sessionId: sessionId,
- msg: '{"type": "trackpad", "x": "$x", "y": "$y", "scale": "$scale"}');
+ msg: '{"type": "trackpad", "x": "$x", "y": "$y"}');
}
}
@@ -427,6 +438,12 @@ class InputModel {
}
void onPointerPanZoomEnd(PointerPanZoomEndEvent e) {
+ bind.sessionSendPointer(
+ sessionId: sessionId,
+ msg: json.encode({
+ 'touch': {'scale': 0}
+ }));
+
waitLastFlingDone();
_stopFling = false;
diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto
index 20eb08ba5..433e186d5 100644
--- a/libs/hbb_common/protos/message.proto
+++ b/libs/hbb_common/protos/message.proto
@@ -111,12 +111,31 @@ message LoginResponse {
}
}
+message TouchScaleUpdate {
+ // The delta scale factor relative to the previous scale.
+ // delta * 1000
+ // 0 means scale end
+ int32 scale = 1;
+}
+
+message TouchEvent {
+ oneof union {
+ TouchScaleUpdate scale_update = 1;
+ }
+ repeated ControlKey modifiers = 2;
+}
+
+message PointerDeviceEvent {
+ oneof union {
+ TouchEvent touch_event = 1;
+ }
+}
+
message MouseEvent {
int32 mask = 1;
sint32 x = 2;
sint32 y = 3;
repeated ControlKey modifiers = 4;
- sint32 scale = 5;
}
enum KeyboardMode{
@@ -683,5 +702,6 @@ message Message {
VoiceCallRequest voice_call_request = 23;
VoiceCallResponse voice_call_response = 24;
PeerInfo peer_info = 25;
+ PointerDeviceEvent pointer_device_event = 26;
}
}
diff --git a/src/client.rs b/src/client.rs
index 6937816a4..798bc83f2 100644
--- a/src/client.rs
+++ b/src/client.rs
@@ -1929,7 +1929,6 @@ pub fn send_mouse(
mask: i32,
x: i32,
y: i32,
- scale: i32,
alt: bool,
ctrl: bool,
shift: bool,
@@ -1941,7 +1940,6 @@ pub fn send_mouse(
mask,
x,
y,
- scale,
..Default::default()
};
if alt {
@@ -1968,18 +1966,54 @@ pub fn send_mouse(
interface.send(Data::Message(msg_out));
}
+#[inline]
+pub fn send_touch(
+ mut evt: TouchEvent,
+ alt: bool,
+ ctrl: bool,
+ shift: bool,
+ command: bool,
+ interface: &impl Interface,
+) {
+ let mut msg_out = Message::new();
+ if alt {
+ evt.modifiers.push(ControlKey::Alt.into());
+ }
+ if shift {
+ evt.modifiers.push(ControlKey::Shift.into());
+ }
+ if ctrl {
+ evt.modifiers.push(ControlKey::Control.into());
+ }
+ if command {
+ evt.modifiers.push(ControlKey::Meta.into());
+ }
+ #[cfg(all(target_os = "macos", not(feature = "flutter")))]
+ if check_scroll_on_mac(mask, x, y) {
+ let factor = 3;
+ mouse_event.mask = crate::input::MOUSE_TYPE_TRACKPAD;
+ mouse_event.x *= factor;
+ mouse_event.y *= factor;
+ }
+ msg_out.set_pointer_device_event(PointerDeviceEvent {
+ union: Some(pointer_device_event::Union::TouchEvent(evt)),
+ ..Default::default()
+ });
+ interface.send(Data::Message(msg_out));
+}
+
/// Activate OS by sending mouse movement.
///
/// # Arguments
///
/// * `interface` - The interface for sending data.
fn activate_os(interface: &impl Interface) {
- send_mouse(0, 0, 0, 0, false, false, false, false, interface);
+ send_mouse(0, 0, 0, false, false, false, false, interface);
std::thread::sleep(Duration::from_millis(50));
- send_mouse(0, 3, 3, 0, false, false, false, false, interface);
+ send_mouse(0, 3, 3, false, false, false, false, interface);
std::thread::sleep(Duration::from_millis(50));
- send_mouse(1 | 1 << 3, 0, 0, 0, false, false, false, false, interface);
- send_mouse(2 | 1 << 3, 0, 0, 0, false, false, false, false, interface);
+ send_mouse(1 | 1 << 3, 0, 0, false, false, false, false, interface);
+ send_mouse(2 | 1 << 3, 0, 0, false, false, false, false, interface);
/*
let mut key_event = KeyEvent::new();
// do not use Esc, which has problem with Linux
diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs
index e4564f144..011c1b098 100644
--- a/src/flutter_ffi.rs
+++ b/src/flutter_ffi.rs
@@ -1069,6 +1069,24 @@ pub fn main_start_dbus_server() {
}
}
+pub fn session_send_pointer(session_id: SessionID, msg: String) {
+ if let Ok(m) = serde_json::from_str::<HashMap<String, serde_json::Value>>(&msg) {
+ let alt = m.get("alt").is_some();
+ let ctrl = m.get("ctrl").is_some();
+ let shift = m.get("shift").is_some();
+ let command = m.get("command").is_some();
+ if let Some(touch_event) = m.get("touch") {
+ if let Some(scale) = touch_event.get("scale") {
+ if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
+ if let Some(scale) = scale.as_i64() {
+ session.send_touch_scale(scale as _, alt, ctrl, shift, command);
+ }
+ }
+ }
+ }
+ }
+}
+
pub fn session_send_mouse(session_id: SessionID, msg: String) {
if let Ok(m) = serde_json::from_str::<HashMap<String, String>>(&msg) {
let alt = m.get("alt").is_some();
@@ -1103,12 +1121,8 @@ pub fn session_send_mouse(session_id: SessionID, msg: String) {
_ => 0,
} << 3;
}
- let scale = m
- .get("scale")
- .map(|x| x.parse::<i32>().unwrap_or(0))
- .unwrap_or(0);
if let Some(session) = SESSIONS.read().unwrap().get(&session_id) {
- session.send_mouse(mask, x, y, scale, alt, ctrl, shift, command);
+ session.send_mouse(mask, x, y, alt, ctrl, shift, command);
}
}
}
diff --git a/src/ipc.rs b/src/ipc.rs
index 526761c8b..9eaade320 100644
--- a/src/ipc.rs
+++ b/src/ipc.rs
@@ -152,6 +152,7 @@ pub enum DataPortableService {
Pong,
ConnCount(Option<usize>),
Mouse((Vec<u8>, i32)),
+ Pointer((Vec<u8>, i32)),
Key(Vec<u8>),
RequestStart,
WillClose,
diff --git a/src/server/connection.rs b/src/server/connection.rs
index 4224d670a..c91b813ff 100644
--- a/src/server/connection.rs
+++ b/src/server/connection.rs
@@ -115,6 +115,8 @@ enum MessageInput {
Mouse((MouseEvent, i32)),
#[cfg(not(any(target_os = "android", target_os = "ios")))]
Key((KeyEvent, bool)),
+ #[cfg(not(any(target_os = "android", target_os = "ios")))]
+ Pointer((PointerDeviceEvent, i32)),
BlockOn,
BlockOff,
#[cfg(all(feature = "flutter", feature = "plugin_framework"))]
@@ -668,6 +670,9 @@ impl Connection {
handle_key(&msg);
}
}
+ MessageInput::Pointer((msg, id)) => {
+ handle_pointer(&msg, id);
+ }
MessageInput::BlockOn => {
if crate::platform::block_input(true) {
block_input_mode = true;
@@ -1179,6 +1184,12 @@ impl Connection {
self.tx_input.send(MessageInput::Mouse((msg, conn_id))).ok();
}
+ #[inline]
+ #[cfg(not(any(target_os = "android", target_os = "ios")))]
+ fn input_pointer(&self, msg: PointerDeviceEvent, conn_id: i32) {
+ self.tx_input.send(MessageInput::Pointer((msg, conn_id))).ok();
+ }
+
#[inline]
#[cfg(not(any(target_os = "android", target_os = "ios")))]
fn input_key(&self, msg: KeyEvent, press: bool) {
@@ -1577,6 +1588,13 @@ impl Connection {
self.input_mouse(me, self.inner.id());
}
}
+ Some(message::Union::PointerDeviceEvent(pde)) => {
+ #[cfg(not(any(target_os = "android", target_os = "ios")))]
+ if self.peer_keyboard_enabled() {
+ MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst);
+ self.input_pointer(pde, self.inner.id());
+ }
+ }
#[cfg(any(target_os = "android", target_os = "ios"))]
Some(message::Union::KeyEvent(..)) => {}
#[cfg(not(any(target_os = "android", target_os = "ios")))]
diff --git a/src/server/input_service.rs b/src/server/input_service.rs
index 49baeab6b..8562ca3eb 100644
--- a/src/server/input_service.rs
+++ b/src/server/input_service.rs
@@ -7,7 +7,11 @@ use crate::input::*;
#[cfg(target_os = "macos")]
use dispatch::Queue;
use enigo::{Enigo, Key, KeyboardControllable, MouseButton, MouseControllable};
-use hbb_common::{get_time, protobuf::EnumOrUnknown};
+use hbb_common::{
+ get_time,
+ message_proto::{pointer_device_event::Union::TouchEvent, touch_event::Union::ScaleUpdate},
+ protobuf::EnumOrUnknown,
+};
use rdev::{self, EventType, Key as RdevKey, KeyCode, RawKey};
#[cfg(target_os = "macos")]
use rdev::{CGEventSourceStateID, CGEventTapLocation, VirtualInput};
@@ -523,6 +527,21 @@ pub fn handle_mouse(evt: &MouseEvent, conn: i32) {
handle_mouse_(evt, conn);
}
+// to-do: merge handle_mouse and handle_pointer
+pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) {
+ #[cfg(target_os = "macos")]
+ if !is_server() {
+ // having GUI, run main GUI thread, otherwise crash
+ let evt = evt.clone();
+ QUEUE.exec_async(move || handle_pointer_(&evt, conn));
+ return;
+ }
+ #[cfg(windows)]
+ crate::portable_service::client::handle_pointer(evt, conn);
+ #[cfg(not(windows))]
+ handle_pointer_(evt, conn);
+}
+
pub fn fix_key_down_timeout_loop() {
std::thread::spawn(move || loop {
std::thread::sleep(std::time::Duration::from_millis(10_000));
@@ -743,7 +762,7 @@ fn active_mouse_(conn: i32) -> bool {
}
}
-pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
+pub fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) {
if !active_mouse_(conn) {
return;
}
@@ -752,12 +771,25 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
return;
}
- if evt.scale != 0 {
- #[cfg(target_os = "windows")]
- {
- handle_scale(evt.scale);
- return;
- }
+ match &evt.union {
+ Some(TouchEvent(evt)) => match &evt.union {
+ Some(ScaleUpdate(_scale_evt)) => {
+ #[cfg(target_os = "windows")]
+ handle_scale(_scale_evt.scale);
+ }
+ _ => {}
+ },
+ _ => {}
+ }
+}
+
+pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
+ if !active_mouse_(conn) {
+ return;
+ }
+
+ if EXITING.load(Ordering::SeqCst) {
+ return;
}
#[cfg(windows)]
@@ -896,10 +928,13 @@ pub fn handle_mouse_(evt: &MouseEvent, conn: i32) {
#[cfg(target_os = "windows")]
fn handle_scale(scale: i32) {
let mut en = ENIGO.lock().unwrap();
- if en.key_down(Key::Control).is_ok() {
- en.mouse_scroll_y(scale);
+ if scale == 0 {
+ en.key_up(Key::Control);
+ } else {
+ if en.key_down(Key::Control).is_ok() {
+ en.mouse_scroll_y(scale);
+ }
}
- en.key_up(Key::Control);
}
pub fn is_enter(evt: &KeyEvent) -> bool {
diff --git a/src/server/portable_service.rs b/src/server/portable_service.rs
index b794e9823..c8185b709 100644
--- a/src/server/portable_service.rs
+++ b/src/server/portable_service.rs
@@ -222,6 +222,8 @@ mod utils {
// functions called in separate SYSTEM user process.
pub mod server {
+ use hbb_common::message_proto::PointerDeviceEvent;
+
use super::*;
lazy_static::lazy_static! {
@@ -466,6 +468,11 @@ pub mod server {
crate::input_service::handle_mouse_(&evt, conn);
}
}
+ Pointer((v, conn)) => {
+ if let Ok(evt) = PointerDeviceEvent::parse_from_bytes(&v) {
+ crate::input_service::handle_pointer_(&evt, conn);
+ }
+ }
Key(v) => {
if let Ok(evt) = KeyEvent::parse_from_bytes(&v) {
crate::input_service::handle_key_(&evt);
@@ -499,7 +506,7 @@ pub mod server {
// functions called in main process.
pub mod client {
- use hbb_common::anyhow::Context;
+ use hbb_common::{anyhow::Context, message_proto::PointerDeviceEvent};
use super::*;
@@ -864,6 +871,14 @@ pub mod client {
))))
}
+ fn handle_pointer_(evt: &PointerDeviceEvent, conn: i32) -> ResultType<()> {
+ let mut v = vec![];
+ evt.write_to_vec(&mut v)?;
+ ipc_send(Data::DataPortableService(DataPortableService::Pointer((
+ v, conn,
+ ))))
+ }
+
fn handle_key_(evt: &KeyEvent) -> ResultType<()> {
let mut v = vec![];
evt.write_to_vec(&mut v)?;
@@ -910,6 +925,15 @@ pub mod client {
}
}
+ pub fn handle_pointer(evt: &PointerDeviceEvent, conn: i32) {
+ if RUNNING.lock().unwrap().clone() {
+ crate::input_service::update_latest_input_cursor_time(conn);
+ handle_pointer_(evt, conn).ok();
+ } else {
+ crate::input_service::handle_pointer_(evt, conn);
+ }
+ }
+
pub fn handle_key(evt: &KeyEvent) {
if RUNNING.lock().unwrap().clone() {
handle_key_(evt).ok();
diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs
index 773eee374..033033faf 100644
--- a/src/ui_session_interface.rs
+++ b/src/ui_session_interface.rs
@@ -35,8 +35,8 @@ use hbb_common::{
use crate::client::io_loop::Remote;
use crate::client::{
check_if_retry, handle_hash, handle_login_error, handle_login_from_ui, handle_test_delay,
- input_os_password, load_config, send_mouse, start_video_audio_threads, FileManager, Key,
- LoginConfigHandler, QualityStatus, KEY_MAP,
+ input_os_password, load_config, send_mouse, send_touch, start_video_audio_threads, FileManager,
+ Key, LoginConfigHandler, QualityStatus, KEY_MAP,
};
#[cfg(not(any(target_os = "android", target_os = "ios")))]
use crate::common::GrabState;
@@ -690,12 +690,23 @@ impl<T: InvokeUiSession> Session<T> {
self.send_key_event(&key_event);
}
+ pub fn send_touch_scale(&self, scale: i32, alt: bool, ctrl: bool, shift: bool, command: bool) {
+ let scale_evt = TouchScaleUpdate {
+ scale,
+ ..Default::default()
+ };
+ let evt = TouchEvent {
+ union: Some(touch_event::Union::ScaleUpdate(scale_evt)),
+ ..Default::default()
+ };
+ send_touch(evt, alt, ctrl, shift, command, self);
+ }
+
pub fn send_mouse(
&self,
mask: i32,
x: i32,
y: i32,
- scale: i32,
alt: bool,
ctrl: bool,
shift: bool,
@@ -714,7 +725,7 @@ impl<T: InvokeUiSession> Session<T> {
let (alt, ctrl, shift, command) =
keyboard::client::get_modifiers_state(alt, ctrl, shift, command);
- send_mouse(mask, x, y, scale, alt, ctrl, shift, command, self);
+ send_mouse(mask, x, y, alt, ctrl, shift, command, self);
// on macos, ctrl + left button down = right button down, up won't emit, so we need to
// emit up myself if peer is not macos
// to-do: how about ctrl + left from win to macos
@@ -730,7 +741,6 @@ impl<T: InvokeUiSession> Session<T> {
(MOUSE_BUTTON_LEFT << 3 | MOUSE_TYPE_UP) as _,
x,
y,
- scale,
alt,
ctrl,
shift,