From 200fc56a4ac787d74475f434ebb20558f46708c9 Mon Sep 17 00:00:00 2001 From: dignow Date: Sun, 6 Aug 2023 14:00:48 +0800 Subject: [PATCH 01/16] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 74 +++++++++++++++++----------- libs/hbb_common/protos/message.proto | 20 ++++++++ 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 6b50aa37f..eeaa1a441 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -592,17 +592,52 @@ class InputModel { return; } evt['type'] = type; + + final pos = handlePointerDevicePos( + x, + y, + isMove, + type, + onExit: onExit, + buttons: evt['buttons'], + ); + if (pos == null) { + return; + } + evt['x'] = '${pos.x}}'; + evt['y'] = '${pos.y}'; + + Map mapButtons = { + kPrimaryMouseButton: 'left', + kSecondaryMouseButton: 'right', + kMiddleMouseButton: 'wheel', + kBackMouseButton: 'back', + kForwardMouseButton: 'forward' + }; + evt['buttons'] = mapButtons[evt['buttons']] ?? ''; + + bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt)); + } + + Point? handlePointerDevicePos( + double x, + double y, + bool isMove, + String evtType, { + bool onExit = false, + int buttons = kPrimaryMouseButton, + }) { y -= CanvasModel.topToEdge; x -= CanvasModel.leftToEdge; final canvasModel = parent.target!.canvasModel; - final nearThr = 3; - var nearRight = (canvasModel.size.width - x) < nearThr; - var nearBottom = (canvasModel.size.height - y) < nearThr; - final ffiModel = parent.target!.ffiModel; if (isMove) { canvasModel.moveDesktopMouse(x, y); } + + final nearThr = 3; + var nearRight = (canvasModel.size.width - x) < nearThr; + var nearBottom = (canvasModel.size.height - y) < nearThr; final d = ffiModel.display; final imageWidth = d.width * canvasModel.scale; final imageHeight = d.height * canvasModel.scale; @@ -650,7 +685,7 @@ class InputModel { } catch (e) { debugPrintStack( label: 'canvasModel.scale value ${canvasModel.scale}, $e'); - return; + return null; } int minX = d.x.toInt(); @@ -661,38 +696,17 @@ class InputModel { evtY = trySetNearestRange(evtY, minY, maxY, 5); if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) { // If left mouse up, no early return. - if (evt['buttons'] != kPrimaryMouseButton || type != 'up') { - return; + if (buttons != kPrimaryMouseButton || evtType != 'up') { + return null; } } - if (type != '') { + if (evtType != '') { evtX = 0; evtY = 0; } - evt['x'] = '$evtX'; - evt['y'] = '$evtY'; - var buttons = ''; - switch (evt['buttons']) { - case kPrimaryMouseButton: - buttons = 'left'; - break; - case kSecondaryMouseButton: - buttons = 'right'; - break; - case kMiddleMouseButton: - buttons = 'wheel'; - break; - case kBackMouseButton: - buttons = 'back'; - break; - case kForwardMouseButton: - buttons = 'forward'; - break; - } - evt['buttons'] = buttons; - bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt)); + return Point(evtX, evtY); } /// Web only diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index e6862bc80..82206cbf2 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -118,9 +118,29 @@ message TouchScaleUpdate { int32 scale = 1; } +message TouchPanStart { + int32 x = 1; + int32 y = 2; +} + +message TouchPanUpdate { + // The delta x position relative to the previous position. + int32 x = 1; + // The delta y position relative to the previous position. + int32 y = 2; +} + +message TouchPanEnd { + int32 x = 1; + int32 y = 2; +} + message TouchEvent { oneof union { TouchScaleUpdate scale_update = 1; + TouchPanStart pan_start = 2; + TouchPanUpdate pan_update = 3; + TouchPanEnd pan_end = 4; } } From 8999bbf2974722ec60d9732b303f322613b4341e Mon Sep 17 00:00:00 2001 From: dignow Date: Tue, 8 Aug 2023 23:53:53 +0800 Subject: [PATCH 02/16] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 71 +++++++++++++++++------------ 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index eeaa1a441..e8cd92a9c 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -223,14 +223,8 @@ class InputModel { command: command); } - Map getEvent(PointerEvent evt, String type) { + Map _getMouseEvent(PointerEvent evt, String type) { final Map out = {}; - out['x'] = evt.position.dx; - out['y'] = evt.position.dy; - if (alt) out['alt'] = 'true'; - if (shift) out['shift'] = 'true'; - if (ctrl) out['ctrl'] = 'true'; - if (command) out['command'] = 'true'; // Check update event type and set buttons to be sent. int buttons = _lastButtons; @@ -260,7 +254,6 @@ class InputModel { out['buttons'] = buttons; out['type'] = type; - return out; } @@ -292,7 +285,7 @@ class InputModel { } /// Modify the given modifier map [evt] based on current modifier key status. - Map modify(Map evt) { + Map modify(Map evt) { if (ctrl) evt['ctrl'] = 'true'; if (shift) evt['shift'] = 'true'; if (alt) evt['alt'] = 'true'; @@ -334,13 +327,14 @@ class InputModel { isPhysicalMouse.value = true; } if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventMove)); + handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position); } } void onPointerPanZoomStart(PointerPanZoomStartEvent e) { _lastScale = 1.0; _stopFling = true; + handlePointerEvent('touch', 'pan_start', e); } // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures @@ -465,21 +459,21 @@ class InputModel { } } if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventDown)); + handleMouse(_getMouseEvent(e, _kMouseEventDown), e.position); } } void onPointUpImage(PointerUpEvent e) { if (e.kind != ui.PointerDeviceKind.mouse) return; if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventUp)); + handleMouse(_getMouseEvent(e, _kMouseEventUp), e.position); } } void onPointMoveImage(PointerMoveEvent e) { if (e.kind != ui.PointerDeviceKind.mouse) return; if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventMove)); + handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position); } } @@ -504,19 +498,16 @@ class InputModel { } void refreshMousePos() => handleMouse({ - 'x': lastMousePos.dx, - 'y': lastMousePos.dy, 'buttons': 0, 'type': _kMouseEventMove, - }); + }, lastMousePos); void tryMoveEdgeOnExit(Offset pos) => handleMouse( { - 'x': pos.dx, - 'y': pos.dy, 'buttons': 0, 'type': _kMouseEventMove, }, + pos, onExit: true, ); @@ -550,17 +541,27 @@ class InputModel { return Offset(x, y); } - void handleMouse( - Map evt, { - bool onExit = false, - }) { - double x = evt['x']; - double y = max(0.0, evt['y']); - final cursorModel = parent.target!.cursorModel; + void handlePointerEvent(String kind, String type, PointerEvent e) { + if (checkPeerControlProtected(e.position.dx, max(0.0, e.position.dy))) { + return; + } + // Only touch events are handled for now. So we can just ignore buttons. + // to-do: handle mouse events + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + modify({ + kind: {type: e.position.toString()} + }) + }), + ); + } + bool checkPeerControlProtected(double x, double y) { + final cursorModel = parent.target!.cursorModel; if (cursorModel.isPeerControlProtected) { lastMousePos = ui.Offset(x, y); - return; + return true; } if (!cursorModel.gotMouseControl) { @@ -571,10 +572,23 @@ class InputModel { cursorModel.gotMouseControl = true; } else { lastMousePos = ui.Offset(x, y); - return; + return true; } } lastMousePos = ui.Offset(x, y); + return false; + } + + void handleMouse( + Map evt, + Offset offset, { + bool onExit = false, + }) { + double x = offset.dx; + double y = max(0.0, offset.dy); + if (checkPeerControlProtected(x, y)) { + return; + } var type = ''; var isMove = false; @@ -615,8 +629,7 @@ class InputModel { kForwardMouseButton: 'forward' }; evt['buttons'] = mapButtons[evt['buttons']] ?? ''; - - bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt)); + bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt))); } Point? handlePointerDevicePos( From 933c99110ccc3e607e70f2ba97211266b5fcf5eb Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 07:39:16 +0800 Subject: [PATCH 03/16] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 85 ++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index e8cd92a9c..ef1a90d6e 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -62,11 +62,11 @@ class InputModel { int _lastButtons = 0; Offset lastMousePos = Offset.zero; - get id => parent.target?.id ?? ""; - late final SessionID sessionId; bool get keyboardPerm => parent.target!.ffiModel.keyboard; + String get id => parent.target?.id ?? ''; + String? get peerPlatform => parent.target?.ffiModel.pi.platform; InputModel(this.parent) { sessionId = parent.target!.sessionId; @@ -334,21 +334,26 @@ class InputModel { void onPointerPanZoomStart(PointerPanZoomStartEvent e) { _lastScale = 1.0; _stopFling = true; - handlePointerEvent('touch', 'pan_start', e); + + if (peerPlatform == kPeerPlatformAndroid) { + handlePointerEvent('touch', 'pan_start', e.position); + } } // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { - final scale = ((e.scale - _lastScale) * 1000).toInt(); - _lastScale = e.scale; + if (peerPlatform != kPeerPlatformAndroid) { + final scale = ((e.scale - _lastScale) * 1000).toInt(); + _lastScale = e.scale; - if (scale != 0) { - bind.sessionSendPointer( - sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': scale} - })); - return; + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + return; + } } final delta = e.panDelta; @@ -356,7 +361,8 @@ class InputModel { var x = delta.dx.toInt(); var y = delta.dy.toInt(); - if (parent.target?.ffiModel.pi.platform == kPeerPlatformLinux) { + if (peerPlatform == kPeerPlatformLinux || + peerPlatform == kPeerPlatformAndroid) { _trackpadScrollUnsent += (delta * _trackpadSpeed); x = _trackpadScrollUnsent.dx.truncate(); y = _trackpadScrollUnsent.dy.truncate(); @@ -372,9 +378,13 @@ class InputModel { } } if (x != 0 || y != 0) { - bind.sessionSendMouse( - sessionId: sessionId, - msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); + if (peerPlatform == kPeerPlatformAndroid) { + handlePointerEvent('touch', 'pan_move', e.delta); + } else { + bind.sessionSendMouse( + sessionId: sessionId, + msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); + } } } @@ -430,6 +440,11 @@ class InputModel { } void onPointerPanZoomEnd(PointerPanZoomEndEvent e) { + if (peerPlatform == kPeerPlatformAndroid) { + handlePointerEvent('touch', 'pan_end', e.position); + return; + } + bind.sessionSendPointer( sessionId: sessionId, msg: json.encode({ @@ -541,23 +556,39 @@ class InputModel { return Offset(x, y); } - void handlePointerEvent(String kind, String type, PointerEvent e) { - if (checkPeerControlProtected(e.position.dx, max(0.0, e.position.dy))) { + void handlePointerEvent(String kind, String type, Offset offset) { + double x = offset.dx; + double y = max(0.0, offset.dy); + if (_checkPeerControlProtected(x, y)) { return; } // Only touch events are handled for now. So we can just ignore buttons. // to-do: handle mouse events - bind.sessionSendPointer( - sessionId: sessionId, - msg: json.encode({ - modify({ - kind: {type: e.position.toString()} - }) - }), + + final isMoveTypes = ['pan', 'pan_start', 'pan_end']; + final pos = handlePointerDevicePos( + x, + y, + isMoveTypes.contains(type), + type, ); + if (pos == null) { + return; + } + + final evt = { + kind: { + type: { + 'x': '${pos.x}', + 'y': '${pos.y}', + } + } + }; + bind.sessionSendPointer( + sessionId: sessionId, msg: json.encode({modify(evt)})); } - bool checkPeerControlProtected(double x, double y) { + bool _checkPeerControlProtected(double x, double y) { final cursorModel = parent.target!.cursorModel; if (cursorModel.isPeerControlProtected) { lastMousePos = ui.Offset(x, y); @@ -586,7 +617,7 @@ class InputModel { }) { double x = offset.dx; double y = max(0.0, offset.dy); - if (checkPeerControlProtected(x, y)) { + if (_checkPeerControlProtected(x, y)) { return; } From d6f1abad959e21e4dbdc4cb05c1a976083b8bb2c Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 09:00:23 +0800 Subject: [PATCH 04/16] tmp commit Signed-off-by: dignow --- .../com/carriez/flutter_hbb/InputService.kt | 31 ++++++++++++++ .../com/carriez/flutter_hbb/MainService.kt | 17 ++++++-- libs/scrap/src/android/ffi.rs | 6 +-- src/server/connection.rs | 42 +++++++++++++++++-- 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 905a2734d..8e14a590d 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -26,6 +26,13 @@ const val WHEEL_BUTTON_UP = 34 const val WHEEL_DOWN = 523331 const val WHEEL_UP = 963 +const val TOUCH_SCALE_START = 1 +const val TOUCH_SCALE = 2 +const val TOUCH_SCALE_END = 3 +const val TOUCH_PAN_START = 4 +const val TOUCH_PAN_UPDATE = 5 +const val TOUCH_PAN_END = 6 + const val WHEEL_STEP = 120 const val WHEEL_DURATION = 50L const val LONG_TAP_DELAY = 200L @@ -167,6 +174,30 @@ class InputService : AccessibilityService() { } } + @RequiresApi(Build.VERSION_CODES.N) + fun onTouchInput(mask: Int, _x: Int, _y: Int) { + val x = max(0, _x) + val y = max(0, _y) + when (mask) { + TOUCH_PAN_UPDATE -> { + mouseX += x * SCREEN_INFO.scale + mouseY += y * SCREEN_INFO.scale + continueGesture(mouseX, mouseY) + } + TOUCH_PAN_START -> { + mouseX = x * SCREEN_INFO.scale + mouseY = y * SCREEN_INFO.scale + startGesture(mouseX, mouseY) + } + TOUCH_PAN_END -> { + mouseX = x * SCREEN_INFO.scale + mouseY = y * SCREEN_INFO.scale + endGesture(mouseX, mouseY) + } + else -> {} + } + } + @RequiresApi(Build.VERSION_CODES.N) private fun consumeWheelActions() { if (isWheelActionsPolling) { diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index 78e4e451e..f32203703 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -71,17 +71,26 @@ class MainService : Service() { @Keep @RequiresApi(Build.VERSION_CODES.N) - fun rustMouseInput(mask: Int, x: Int, y: Int) { + fun rustPointerInput(kind: String, mask: Int, x: Int, y: Int) { // turn on screen with LIFT_DOWN when screen off - if (!powerManager.isInteractive && mask == LIFT_DOWN) { + if (!powerManager.isInteractive && (kind == "touch" || mask == LIFT_DOWN)) { if (wakeLock.isHeld) { - Log.d(logTag,"Turn on Screen, WakeLock release") + Log.d(logTag, "Turn on Screen, WakeLock release") wakeLock.release() } Log.d(logTag,"Turn on Screen") wakeLock.acquire(5000) } else { - InputService.ctx?.onMouseInput(mask,x,y) + when (name) { + "touch" -> { + InputService.ctx?.onTouchInput(mask, x, y) + } + "mouse" -> { + InputService.ctx?.onMouseInput(mask, x, y) + } + else -> { + } + } } } diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 6855fd3f6..3801bbc87 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -154,7 +154,7 @@ pub extern "system" fn Java_com_carriez_flutter_1hbb_MainService_init( } } -pub fn call_main_service_mouse_input(mask: i32, x: i32, y: i32) -> JniResult<()> { +pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), MAIN_SERVICE_CTX.read().unwrap().as_ref(), @@ -162,9 +162,9 @@ pub fn call_main_service_mouse_input(mask: i32, x: i32, y: i32) -> JniResult<()> let mut env = jvm.attach_current_thread_as_daemon()?; env.call_method( ctx, - "rustMouseInput", + "rustPointerInput", "(III)V", - &[JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + &[JValue(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { diff --git a/src/server/connection.rs b/src/server/connection.rs index e32a4c1c3..913c831da 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1546,8 +1546,10 @@ impl Connection { match msg.union { Some(message::Union::MouseEvent(me)) => { #[cfg(any(target_os = "android", target_os = "ios"))] - if let Err(e) = call_main_service_mouse_input(me.mask, me.x, me.y) { - log::debug!("call_main_service_mouse_input fail:{}", e); + if let Err(e) = + call_main_service_pointer_input("mouse".to_string(), me.mask, me.x, me.y) + { + log::debug!("call_main_service_pointer_input fail:{}", e); } #[cfg(not(any(target_os = "android", target_os = "ios")))] if self.peer_keyboard_enabled() { @@ -1559,8 +1561,40 @@ impl Connection { self.input_mouse(me, self.inner.id()); } } - Some(message::Union::PointerDeviceEvent(pde)) => - { + Some(message::Union::PointerDeviceEvent(pde)) => { + #[cfg(any(target_os = "android", target_os = "ios"))] + if let Err(e) = match pde.union { + Some(pointer_device_event::Union::TouchEvent(touch)) => match touch.union { + Some(touch_event::Union::PanStart(pan_start)) => { + call_main_service_pointer_input( + "touch".to_string(), + 4, + pan_start.x, + pan_start.y, + ) + } + Some(touch_event::Union::PanUpdate(pan_start)) => { + call_main_service_pointer_input( + "touch".to_string(), + 5, + pan_start.x, + pan_start.y, + ) + } + Some(touch_event::Union::PanEnd(pan_start)) => { + call_main_service_pointer_input( + "touch".to_string(), + 6, + pan_start.x, + pan_start.y, + ) + } + _ => Ok(()), + }, + _ => Ok(()), + } { + log::debug!("call_main_service_pointer_input fail:{}", e); + } #[cfg(not(any(target_os = "android", target_os = "ios")))] if self.peer_keyboard_enabled() { MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst); From 06ee68f836a4e5635ec41fc460afe39d5e851f41 Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 22:26:13 +0800 Subject: [PATCH 05/16] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index ef1a90d6e..18e9d0d2f 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -649,7 +649,7 @@ class InputModel { if (pos == null) { return; } - evt['x'] = '${pos.x}}'; + evt['x'] = '${pos.x}'; evt['y'] = '${pos.y}'; Map mapButtons = { From 93a600a0a8d86cfdb632f15552cb0b887448e0fc Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 23:42:53 +0800 Subject: [PATCH 06/16] tmp commit Signed-off-by: dignow --- flutter/lib/common/widgets/remote_input.dart | 12 +-- flutter/lib/consts.dart | 3 + flutter/lib/models/input_model.dart | 101 ++++++++++++------- src/flutter_ffi.rs | 41 ++++++-- src/server/connection.rs | 12 +-- src/ui_session_interface.rs | 43 ++++++++ 6 files changed, 155 insertions(+), 57 deletions(-) diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index 35eaf8a7c..b00cd1fb4 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -6,6 +6,7 @@ import 'package:flutter/gestures.dart'; import 'package:flutter_hbb/models/platform_model.dart'; import 'package:flutter_hbb/common.dart'; +import 'package:flutter_hbb/consts.dart'; import 'package:flutter_hbb/models/model.dart'; import 'package:flutter_hbb/models/input_model.dart'; @@ -263,9 +264,9 @@ class _RawTouchGestureDetectorRegionState if (scale != 0) { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': scale} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', scale) + .toJson())); } } else { // mobile @@ -283,9 +284,8 @@ class _RawTouchGestureDetectorRegionState if (isDesktop) { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': 0} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson())); } else { // mobile _scale = 1; diff --git a/flutter/lib/consts.dart b/flutter/lib/consts.dart index 5376196e4..f26e83e01 100644 --- a/flutter/lib/consts.dart +++ b/flutter/lib/consts.dart @@ -54,6 +54,9 @@ const String kTabLabelSettingPage = "Settings"; const String kWindowPrefix = "wm_"; const int kWindowMainId = 0; +const String kPointerEventKindTouch = "touch"; +const String kPointerEventKindMouse = "mouse"; + // the executable name of the portable version const String kEnvPortableExecutable = "RUSTDESK_APPNAME"; diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 18e9d0d2f..9b1ec4437 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -35,6 +35,24 @@ extension ToString on MouseButtons { } } +class PointerEventToRust { + final String kind; + final String type; + final dynamic value; + + PointerEventToRust(this.kind, this.type, this.value); + + Map toJson() { + return { + 'k': kind, + 'v': { + 't': type, + 'v': value, + } + }; + } +} + class InputModel { final WeakReference parent; String keyboardMode = "legacy"; @@ -349,9 +367,9 @@ class InputModel { if (scale != 0) { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': scale} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', scale) + .toJson())); return; } } @@ -379,7 +397,7 @@ class InputModel { } if (x != 0 || y != 0) { if (peerPlatform == kPeerPlatformAndroid) { - handlePointerEvent('touch', 'pan_move', e.delta); + handlePointerEvent('touch', 'pan_update', Offset(x.toDouble(), y.toDouble())); } else { bind.sessionSendMouse( sessionId: sessionId, @@ -447,9 +465,8 @@ class InputModel { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': 0} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson())); waitLastFlingDone(); _stopFling = false; @@ -558,34 +575,40 @@ class InputModel { void handlePointerEvent(String kind, String type, Offset offset) { double x = offset.dx; - double y = max(0.0, offset.dy); + double y = offset.dy; if (_checkPeerControlProtected(x, y)) { return; } // Only touch events are handled for now. So we can just ignore buttons. // to-do: handle mouse events - final isMoveTypes = ['pan', 'pan_start', 'pan_end']; - final pos = handlePointerDevicePos( - x, - y, - isMoveTypes.contains(type), - type, - ); - if (pos == null) { - return; + late final dynamic evtValue; + if (type == 'pan_update') { + evtValue = { + 'x': '${x.toInt()}', + 'y': '${y.toInt()}', + }; + } else { + final isMoveTypes = ['pan_start', 'pan_end']; + final pos = handlePointerDevicePos( + kPointerEventKindTouch, + x, + y, + isMoveTypes.contains(type), + type, + ); + if (pos == null) { + return; + } + evtValue = { + 'x': '${pos.x}', + 'y': '${pos.y}', + }; } - final evt = { - kind: { - type: { - 'x': '${pos.x}', - 'y': '${pos.y}', - } - } - }; + final evt = PointerEventToRust(kind, type, evtValue).toJson(); bind.sessionSendPointer( - sessionId: sessionId, msg: json.encode({modify(evt)})); + sessionId: sessionId, msg: json.encode(modify(evt))); } bool _checkPeerControlProtected(double x, double y) { @@ -639,6 +662,7 @@ class InputModel { evt['type'] = type; final pos = handlePointerDevicePos( + kPointerEventKindMouse, x, y, isMove, @@ -649,8 +673,13 @@ class InputModel { if (pos == null) { return; } - evt['x'] = '${pos.x}'; - evt['y'] = '${pos.y}'; + if (type != '') { + evt['x'] = 0; + evt['y'] = 0; + } else { + evt['x'] = '${pos.x}'; + evt['y'] = '${pos.y}'; + } Map mapButtons = { kPrimaryMouseButton: 'left', @@ -664,6 +693,7 @@ class InputModel { } Point? handlePointerDevicePos( + String kind, double x, double y, bool isMove, @@ -738,18 +768,15 @@ class InputModel { int maxY = (d.y + d.height).toInt() - 1; evtX = trySetNearestRange(evtX, minX, maxX, 5); evtY = trySetNearestRange(evtY, minY, maxY, 5); - if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) { - // If left mouse up, no early return. - if (buttons != kPrimaryMouseButton || evtType != 'up') { - return null; + if (kind == kPointerEventKindMouse) { + if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) { + // If left mouse up, no early return. + if (!(buttons == kPrimaryMouseButton && evtType == 'up')) { + return null; + } } } - if (evtType != '') { - evtX = 0; - evtY = 0; - } - return Point(evtX, evtY); } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index ef4c9e2ba..cf5d23397 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1177,14 +1177,39 @@ pub fn session_send_pointer(session_id: SessionID, msg: String) { let ctrl = m.get("ctrl").is_some(); let shift = m.get("shift").is_some(); let command = m.get("command").is_some(); - if let Some(touch_event) = m.get("touch") { - if let Some(scale) = touch_event.get("scale") { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - if let Some(scale) = scale.as_i64() { - session.send_touch_scale(scale as _, alt, ctrl, shift, command); - } - } - } + match (m.get("k"), m.get("v")) { + (Some(k), Some(v)) => match k.as_str() { + Some("touch") => match v.as_str() { + Some("scale") => match v.get("v") { + Some(scale) => { + if let Some(scale) = scale.as_i64() { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + } + None => {} + }, + Some(pan_event) => match (v.get("x"), v.get("y")) { + (Some(x), Some(y)) => { + if let Some(x) = x.as_i64() { + if let Some(y) = y.as_i64() { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) + { + session.send_touch_pan_event( + pan_event, x as _, y as _, alt, ctrl, shift, command, + ); + } + } + } + } + _ => {} + }, + _ => {} + }, + _ => {} + }, + _ => {} } } } diff --git a/src/server/connection.rs b/src/server/connection.rs index 913c831da..972750fc2 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1573,20 +1573,20 @@ impl Connection { pan_start.y, ) } - Some(touch_event::Union::PanUpdate(pan_start)) => { + Some(touch_event::Union::PanUpdate(pan_update)) => { call_main_service_pointer_input( "touch".to_string(), 5, - pan_start.x, - pan_start.y, + pan_update.x, + pan_update.y, ) } - Some(touch_event::Union::PanEnd(pan_start)) => { + Some(touch_event::Union::PanEnd(pan_end)) => { call_main_service_pointer_input( "touch".to_string(), 6, - pan_start.x, - pan_start.y, + pan_end.x, + pan_end.y, ) } _ => Ok(()), diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 7ffcaac6d..23035a550 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -724,6 +724,49 @@ impl Session { send_pointer_device_event(evt, alt, ctrl, shift, command, self); } + pub fn send_touch_pan_event( + &self, + event: &str, + x: i32, + y: i32, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, + ) { + let mut touch_evt = TouchEvent::new(); + match event { + "pan_start" => { + touch_evt.set_pan_start(TouchPanStart { + x, + y, + ..Default::default() + }); + } + "pan_update" => { + touch_evt.set_pan_update(TouchPanUpdate { + x, + y, + ..Default::default() + }); + } + "pan_end" => { + touch_evt.set_pan_end(TouchPanEnd { + x, + y, + ..Default::default() + }); + } + _ => { + log::warn!("unknown touch pan event: {}", event); + return; + } + }; + let mut evt = PointerDeviceEvent::new(); + evt.set_touch_event(touch_evt); + send_pointer_device_event(evt, alt, ctrl, shift, command, self); + } + pub fn send_mouse( &self, mask: i32, From 9e0feb0b64c14c97eccea3905923dbcc7acbaab5 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 01:02:27 +0800 Subject: [PATCH 07/16] tmp debug Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 8 +++---- src/flutter_ffi.rs | 33 ++++++++++++++--------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 9b1ec4437..5f851c54d 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -585,8 +585,8 @@ class InputModel { late final dynamic evtValue; if (type == 'pan_update') { evtValue = { - 'x': '${x.toInt()}', - 'y': '${y.toInt()}', + 'x': x.toInt(), + 'y': y.toInt(), }; } else { final isMoveTypes = ['pan_start', 'pan_end']; @@ -601,8 +601,8 @@ class InputModel { return; } evtValue = { - 'x': '${pos.x}', - 'y': '${pos.y}', + 'x': pos.x, + 'y': pos.y, }; } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index cf5d23397..c6f8ca774 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1179,30 +1179,29 @@ pub fn session_send_pointer(session_id: SessionID, msg: String) { let command = m.get("command").is_some(); match (m.get("k"), m.get("v")) { (Some(k), Some(v)) => match k.as_str() { - Some("touch") => match v.as_str() { - Some("scale") => match v.get("v") { + Some("touch") => match v.get("t").and_then(|t| t.as_str()) { + Some("scale") => match v.get("v").and_then(|s| s.as_i64()) { Some(scale) => { - if let Some(scale) = scale.as_i64() { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_touch_scale(scale as _, alt, ctrl, shift, command); - } + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); } } None => {} }, - Some(pan_event) => match (v.get("x"), v.get("y")) { - (Some(x), Some(y)) => { - if let Some(x) = x.as_i64() { - if let Some(y) = y.as_i64() { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) - { - session.send_touch_pan_event( - pan_event, x as _, y as _, alt, ctrl, shift, command, - ); - } + Some(pan_event) => match v.get("v") { + Some(v) => match ( + v.get("x").and_then(|x| x.as_i64()), + v.get("y").and_then(|y| y.as_i64()), + ) { + (Some(x), Some(y)) => { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_pan_event( + pan_event, x as _, y as _, alt, ctrl, shift, command, + ); } } - } + _ => {} + }, _ => {} }, _ => {} From da16a799fa6851289bc18b081cfa4e715e6163f3 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 01:10:59 +0800 Subject: [PATCH 08/16] fix build Signed-off-by: dignow --- libs/scrap/src/android/ffi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 3801bbc87..78908a2ba 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -164,7 +164,7 @@ pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) ctx, "rustPointerInput", "(III)V", - &[JValue(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + &[JValue::String(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { From 9476d7fdbb7edb3b3ebcaa8326622918b71af227 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 01:24:39 +0800 Subject: [PATCH 09/16] try fix build Signed-off-by: dignow --- libs/scrap/src/android/ffi.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 78908a2ba..ba775dfd3 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -163,8 +163,8 @@ pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) env.call_method( ctx, "rustPointerInput", - "(III)V", - &[JValue::String(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + "(Ljava/lang/String;III)V", + &[JValue::Object(&JObject::from(name)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { From e89ae475f6a4d03ca79d0382e71d1fb9ea816b40 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 03:01:46 +0800 Subject: [PATCH 10/16] fix build Signed-off-by: dignow --- .../main/kotlin/com/carriez/flutter_hbb/MainService.kt | 2 +- libs/scrap/src/android/ffi.rs | 5 +++-- src/server/connection.rs | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index f32203703..535a3f8c3 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -81,7 +81,7 @@ class MainService : Service() { Log.d(logTag,"Turn on Screen") wakeLock.acquire(5000) } else { - when (name) { + when (kind) { "touch" -> { InputService.ctx?.onTouchInput(mask, x, y) } diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index ba775dfd3..e9c60ef93 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -154,17 +154,18 @@ pub extern "system" fn Java_com_carriez_flutter_1hbb_MainService_init( } } -pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) -> JniResult<()> { +pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), MAIN_SERVICE_CTX.read().unwrap().as_ref(), ) { let mut env = jvm.attach_current_thread_as_daemon()?; + let kind = env.new_string(kind)?; env.call_method( ctx, "rustPointerInput", "(Ljava/lang/String;III)V", - &[JValue::Object(&JObject::from(name)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + &[JValue::Object(&JObject::from(kind)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { diff --git a/src/server/connection.rs b/src/server/connection.rs index 972750fc2..f107d15a6 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -39,7 +39,7 @@ use hbb_common::{ tokio_util::codec::{BytesCodec, Framed}, }; #[cfg(any(target_os = "android", target_os = "ios"))] -use scrap::android::call_main_service_mouse_input; +use scrap::android::call_main_service_pointer_input; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1547,7 +1547,7 @@ impl Connection { Some(message::Union::MouseEvent(me)) => { #[cfg(any(target_os = "android", target_os = "ios"))] if let Err(e) = - call_main_service_pointer_input("mouse".to_string(), me.mask, me.x, me.y) + call_main_service_pointer_input("mouse", me.mask, me.x, me.y) { log::debug!("call_main_service_pointer_input fail:{}", e); } @@ -1567,7 +1567,7 @@ impl Connection { Some(pointer_device_event::Union::TouchEvent(touch)) => match touch.union { Some(touch_event::Union::PanStart(pan_start)) => { call_main_service_pointer_input( - "touch".to_string(), + "touch", 4, pan_start.x, pan_start.y, @@ -1575,7 +1575,7 @@ impl Connection { } Some(touch_event::Union::PanUpdate(pan_update)) => { call_main_service_pointer_input( - "touch".to_string(), + "touch", 5, pan_update.x, pan_update.y, @@ -1583,7 +1583,7 @@ impl Connection { } Some(touch_event::Union::PanEnd(pan_end)) => { call_main_service_pointer_input( - "touch".to_string(), + "touch", 6, pan_end.x, pan_end.y, From b9c8df70196ca0eeb9f8115729a3775b1cd38600 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 03:55:03 +0800 Subject: [PATCH 11/16] debug Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 7 +++++-- src/server/connection.rs | 11 ++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 5f851c54d..eaff4ed6a 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -638,6 +638,7 @@ class InputModel { Offset offset, { bool onExit = false, }) { + debugPrint('REMOVE ME ========================= 111'); double x = offset.dx; double y = max(0.0, offset.dy); if (_checkPeerControlProtected(x, y)) { @@ -671,11 +672,12 @@ class InputModel { buttons: evt['buttons'], ); if (pos == null) { + debugPrint('REMOVE ME ========================= 222'); return; } if (type != '') { - evt['x'] = 0; - evt['y'] = 0; + evt['x'] = '0'; + evt['y'] = '0'; } else { evt['x'] = '${pos.x}'; evt['y'] = '${pos.y}'; @@ -689,6 +691,7 @@ class InputModel { kForwardMouseButton: 'forward' }; evt['buttons'] = mapButtons[evt['buttons']] ?? ''; + debugPrint('REMOVE ME ========================= 333 $evt'); bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt))); } diff --git a/src/server/connection.rs b/src/server/connection.rs index f107d15a6..ef8864499 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1546,9 +1546,7 @@ impl Connection { match msg.union { Some(message::Union::MouseEvent(me)) => { #[cfg(any(target_os = "android", target_os = "ios"))] - if let Err(e) = - call_main_service_pointer_input("mouse", me.mask, me.x, me.y) - { + if let Err(e) = call_main_service_pointer_input("mouse", me.mask, me.x, me.y) { log::debug!("call_main_service_pointer_input fail:{}", e); } #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1582,12 +1580,7 @@ impl Connection { ) } Some(touch_event::Union::PanEnd(pan_end)) => { - call_main_service_pointer_input( - "touch", - 6, - pan_end.x, - pan_end.y, - ) + call_main_service_pointer_input("touch", 6, pan_end.x, pan_end.y) } _ => Ok(()), }, From be982d95ea548baeee37d805e198b67a2b97da8a Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 08:37:18 +0800 Subject: [PATCH 12/16] tmp build Signed-off-by: dignow --- .../src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 2 ++ flutter/lib/models/input_model.dart | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 8e14a590d..84ef31793 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -190,8 +190,10 @@ class InputService : AccessibilityService() { startGesture(mouseX, mouseY) } TOUCH_PAN_END -> { + endGesture(mouseX, mouseY) mouseX = x * SCREEN_INFO.scale mouseY = y * SCREEN_INFO.scale + continueGesture(mouseX, mouseY) endGesture(mouseX, mouseY) } else -> {} diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index eaff4ed6a..971bbb7e5 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -379,8 +379,7 @@ class InputModel { var x = delta.dx.toInt(); var y = delta.dy.toInt(); - if (peerPlatform == kPeerPlatformLinux || - peerPlatform == kPeerPlatformAndroid) { + if (peerPlatform == kPeerPlatformLinux) { _trackpadScrollUnsent += (delta * _trackpadSpeed); x = _trackpadScrollUnsent.dx.truncate(); y = _trackpadScrollUnsent.dy.truncate(); @@ -638,7 +637,6 @@ class InputModel { Offset offset, { bool onExit = false, }) { - debugPrint('REMOVE ME ========================= 111'); double x = offset.dx; double y = max(0.0, offset.dy); if (_checkPeerControlProtected(x, y)) { @@ -672,7 +670,6 @@ class InputModel { buttons: evt['buttons'], ); if (pos == null) { - debugPrint('REMOVE ME ========================= 222'); return; } if (type != '') { @@ -691,7 +688,6 @@ class InputModel { kForwardMouseButton: 'forward' }; evt['buttons'] = mapButtons[evt['buttons']] ?? ''; - debugPrint('REMOVE ME ========================= 333 $evt'); bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt))); } From 5f7055e28286174440f7a8ee17ff7bec0d600af2 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 13:57:57 +0800 Subject: [PATCH 13/16] debug Signed-off-by: dignow --- .../src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 84ef31793..65b7931db 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -180,8 +180,8 @@ class InputService : AccessibilityService() { val y = max(0, _y) when (mask) { TOUCH_PAN_UPDATE -> { - mouseX += x * SCREEN_INFO.scale - mouseY += y * SCREEN_INFO.scale + mouseX -= x * SCREEN_INFO.scale + mouseY -= y * SCREEN_INFO.scale continueGesture(mouseX, mouseY) } TOUCH_PAN_START -> { @@ -193,8 +193,6 @@ class InputService : AccessibilityService() { endGesture(mouseX, mouseY) mouseX = x * SCREEN_INFO.scale mouseY = y * SCREEN_INFO.scale - continueGesture(mouseX, mouseY) - endGesture(mouseX, mouseY) } else -> {} } From 072430cef5c0452b4a1ea36b243b1eddc1d0fbdd Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 14:48:00 +0800 Subject: [PATCH 14/16] debug android scroll Signed-off-by: dignow --- .../kotlin/com/carriez/flutter_hbb/InputService.kt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 65b7931db..55c57729e 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -176,23 +176,21 @@ class InputService : AccessibilityService() { @RequiresApi(Build.VERSION_CODES.N) fun onTouchInput(mask: Int, _x: Int, _y: Int) { - val x = max(0, _x) - val y = max(0, _y) when (mask) { TOUCH_PAN_UPDATE -> { - mouseX -= x * SCREEN_INFO.scale - mouseY -= y * SCREEN_INFO.scale + mouseX -= _x * SCREEN_INFO.scale + mouseY -= _y * SCREEN_INFO.scale continueGesture(mouseX, mouseY) } TOUCH_PAN_START -> { - mouseX = x * SCREEN_INFO.scale - mouseY = y * SCREEN_INFO.scale + mouseX = max(0, _x) * SCREEN_INFO.scale + mouseY = max(0, _y) * SCREEN_INFO.scale startGesture(mouseX, mouseY) } TOUCH_PAN_END -> { endGesture(mouseX, mouseY) - mouseX = x * SCREEN_INFO.scale - mouseY = y * SCREEN_INFO.scale + mouseX = max(0, _x) * SCREEN_INFO.scale + mouseY = max(0, _y) * SCREEN_INFO.scale } else -> {} } From 5b2358c97fb40978b30473b034a913c7d6137cce Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 14:49:49 +0800 Subject: [PATCH 15/16] debug android scroll Signed-off-by: dignow --- .../app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 55c57729e..203558968 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -180,6 +180,8 @@ class InputService : AccessibilityService() { TOUCH_PAN_UPDATE -> { mouseX -= _x * SCREEN_INFO.scale mouseY -= _y * SCREEN_INFO.scale + mouseX = max(0, mouseX); + mouseY = max(0, mouseY); continueGesture(mouseX, mouseY) } TOUCH_PAN_START -> { From 6368ab691c47f6292e2156c2f3734fb46f45fa6e Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 16:08:30 +0800 Subject: [PATCH 16/16] simple refactor, move code from flutter_ffi.rs to flutter.rs Signed-off-by: dignow --- src/flutter.rs | 79 ++++++++++++++++++++++++++++++++++++++++++++++ src/flutter_ffi.rs | 40 +---------------------- 2 files changed, 80 insertions(+), 39 deletions(-) diff --git a/src/flutter.rs b/src/flutter.rs index 52190ce2e..32be6b1c3 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -1130,6 +1130,85 @@ pub fn stop_global_event_stream(app_type: String) { let _ = GLOBAL_EVENT_STREAM.write().unwrap().remove(&app_type); } +#[inline] +fn session_send_touch_scale( + session_id: SessionID, + v: &serde_json::Value, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, +) { + match v.get("v").and_then(|s| s.as_i64()) { + Some(scale) => { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + None => {} + } +} + +#[inline] +fn session_send_touch_pan( + session_id: SessionID, + v: &serde_json::Value, + pan_event: &str, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, +) { + match v.get("v") { + Some(v) => match ( + v.get("x").and_then(|x| x.as_i64()), + v.get("y").and_then(|y| y.as_i64()), + ) { + (Some(x), Some(y)) => { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session + .send_touch_pan_event(pan_event, x as _, y as _, alt, ctrl, shift, command); + } + } + _ => {} + }, + _ => {} + } +} + +fn session_send_touch_event( + session_id: SessionID, + v: &serde_json::Value, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, +) { + match v.get("t").and_then(|t| t.as_str()) { + Some("scale") => session_send_touch_scale(session_id, v, alt, ctrl, shift, command), + Some(pan_event) => { + session_send_touch_pan(session_id, v, pan_event, alt, ctrl, shift, command) + } + _ => {} + } +} + +pub fn session_send_pointer(session_id: SessionID, msg: String) { + if let Ok(m) = serde_json::from_str::>(&msg) { + let alt = m.get("alt").is_some(); + let ctrl = m.get("ctrl").is_some(); + let shift = m.get("shift").is_some(); + let command = m.get("command").is_some(); + match (m.get("k"), m.get("v")) { + (Some(k), Some(v)) => match k.as_str() { + Some("touch") => session_send_touch_event(session_id, v, alt, ctrl, shift, command), + _ => {} + }, + _ => {} + } + } +} + #[no_mangle] unsafe extern "C" fn get_rgba() {} diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index c6f8ca774..34a1b61e3 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1172,45 +1172,7 @@ pub fn main_clear_ab() { } pub fn session_send_pointer(session_id: SessionID, msg: String) { - if let Ok(m) = serde_json::from_str::>(&msg) { - let alt = m.get("alt").is_some(); - let ctrl = m.get("ctrl").is_some(); - let shift = m.get("shift").is_some(); - let command = m.get("command").is_some(); - match (m.get("k"), m.get("v")) { - (Some(k), Some(v)) => match k.as_str() { - Some("touch") => match v.get("t").and_then(|t| t.as_str()) { - Some("scale") => match v.get("v").and_then(|s| s.as_i64()) { - Some(scale) => { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_touch_scale(scale as _, alt, ctrl, shift, command); - } - } - None => {} - }, - Some(pan_event) => match v.get("v") { - Some(v) => match ( - v.get("x").and_then(|x| x.as_i64()), - v.get("y").and_then(|y| y.as_i64()), - ) { - (Some(x), Some(y)) => { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_touch_pan_event( - pan_event, x as _, y as _, alt, ctrl, shift, command, - ); - } - } - _ => {} - }, - _ => {} - }, - _ => {} - }, - _ => {} - }, - _ => {} - } - } + super::flutter::session_send_pointer(session_id, msg); } pub fn session_send_mouse(session_id: SessionID, msg: String) {