From 200fc56a4ac787d74475f434ebb20558f46708c9 Mon Sep 17 00:00:00 2001 From: dignow Date: Sun, 6 Aug 2023 14:00:48 +0800 Subject: [PATCH 01/20] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 74 +++++++++++++++++----------- libs/hbb_common/protos/message.proto | 20 ++++++++ 2 files changed, 64 insertions(+), 30 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 6b50aa37f..eeaa1a441 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -592,17 +592,52 @@ class InputModel { return; } evt['type'] = type; + + final pos = handlePointerDevicePos( + x, + y, + isMove, + type, + onExit: onExit, + buttons: evt['buttons'], + ); + if (pos == null) { + return; + } + evt['x'] = '${pos.x}}'; + evt['y'] = '${pos.y}'; + + Map mapButtons = { + kPrimaryMouseButton: 'left', + kSecondaryMouseButton: 'right', + kMiddleMouseButton: 'wheel', + kBackMouseButton: 'back', + kForwardMouseButton: 'forward' + }; + evt['buttons'] = mapButtons[evt['buttons']] ?? ''; + + bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt)); + } + + Point? handlePointerDevicePos( + double x, + double y, + bool isMove, + String evtType, { + bool onExit = false, + int buttons = kPrimaryMouseButton, + }) { y -= CanvasModel.topToEdge; x -= CanvasModel.leftToEdge; final canvasModel = parent.target!.canvasModel; - final nearThr = 3; - var nearRight = (canvasModel.size.width - x) < nearThr; - var nearBottom = (canvasModel.size.height - y) < nearThr; - final ffiModel = parent.target!.ffiModel; if (isMove) { canvasModel.moveDesktopMouse(x, y); } + + final nearThr = 3; + var nearRight = (canvasModel.size.width - x) < nearThr; + var nearBottom = (canvasModel.size.height - y) < nearThr; final d = ffiModel.display; final imageWidth = d.width * canvasModel.scale; final imageHeight = d.height * canvasModel.scale; @@ -650,7 +685,7 @@ class InputModel { } catch (e) { debugPrintStack( label: 'canvasModel.scale value ${canvasModel.scale}, $e'); - return; + return null; } int minX = d.x.toInt(); @@ -661,38 +696,17 @@ class InputModel { evtY = trySetNearestRange(evtY, minY, maxY, 5); if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) { // If left mouse up, no early return. - if (evt['buttons'] != kPrimaryMouseButton || type != 'up') { - return; + if (buttons != kPrimaryMouseButton || evtType != 'up') { + return null; } } - if (type != '') { + if (evtType != '') { evtX = 0; evtY = 0; } - evt['x'] = '$evtX'; - evt['y'] = '$evtY'; - var buttons = ''; - switch (evt['buttons']) { - case kPrimaryMouseButton: - buttons = 'left'; - break; - case kSecondaryMouseButton: - buttons = 'right'; - break; - case kMiddleMouseButton: - buttons = 'wheel'; - break; - case kBackMouseButton: - buttons = 'back'; - break; - case kForwardMouseButton: - buttons = 'forward'; - break; - } - evt['buttons'] = buttons; - bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt)); + return Point(evtX, evtY); } /// Web only diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index e6862bc80..82206cbf2 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -118,9 +118,29 @@ message TouchScaleUpdate { int32 scale = 1; } +message TouchPanStart { + int32 x = 1; + int32 y = 2; +} + +message TouchPanUpdate { + // The delta x position relative to the previous position. + int32 x = 1; + // The delta y position relative to the previous position. + int32 y = 2; +} + +message TouchPanEnd { + int32 x = 1; + int32 y = 2; +} + message TouchEvent { oneof union { TouchScaleUpdate scale_update = 1; + TouchPanStart pan_start = 2; + TouchPanUpdate pan_update = 3; + TouchPanEnd pan_end = 4; } } From 8999bbf2974722ec60d9732b303f322613b4341e Mon Sep 17 00:00:00 2001 From: dignow Date: Tue, 8 Aug 2023 23:53:53 +0800 Subject: [PATCH 02/20] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 71 +++++++++++++++++------------ 1 file changed, 42 insertions(+), 29 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index eeaa1a441..e8cd92a9c 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -223,14 +223,8 @@ class InputModel { command: command); } - Map getEvent(PointerEvent evt, String type) { + Map _getMouseEvent(PointerEvent evt, String type) { final Map out = {}; - out['x'] = evt.position.dx; - out['y'] = evt.position.dy; - if (alt) out['alt'] = 'true'; - if (shift) out['shift'] = 'true'; - if (ctrl) out['ctrl'] = 'true'; - if (command) out['command'] = 'true'; // Check update event type and set buttons to be sent. int buttons = _lastButtons; @@ -260,7 +254,6 @@ class InputModel { out['buttons'] = buttons; out['type'] = type; - return out; } @@ -292,7 +285,7 @@ class InputModel { } /// Modify the given modifier map [evt] based on current modifier key status. - Map modify(Map evt) { + Map modify(Map evt) { if (ctrl) evt['ctrl'] = 'true'; if (shift) evt['shift'] = 'true'; if (alt) evt['alt'] = 'true'; @@ -334,13 +327,14 @@ class InputModel { isPhysicalMouse.value = true; } if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventMove)); + handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position); } } void onPointerPanZoomStart(PointerPanZoomStartEvent e) { _lastScale = 1.0; _stopFling = true; + handlePointerEvent('touch', 'pan_start', e); } // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures @@ -465,21 +459,21 @@ class InputModel { } } if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventDown)); + handleMouse(_getMouseEvent(e, _kMouseEventDown), e.position); } } void onPointUpImage(PointerUpEvent e) { if (e.kind != ui.PointerDeviceKind.mouse) return; if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventUp)); + handleMouse(_getMouseEvent(e, _kMouseEventUp), e.position); } } void onPointMoveImage(PointerMoveEvent e) { if (e.kind != ui.PointerDeviceKind.mouse) return; if (isPhysicalMouse.value) { - handleMouse(getEvent(e, _kMouseEventMove)); + handleMouse(_getMouseEvent(e, _kMouseEventMove), e.position); } } @@ -504,19 +498,16 @@ class InputModel { } void refreshMousePos() => handleMouse({ - 'x': lastMousePos.dx, - 'y': lastMousePos.dy, 'buttons': 0, 'type': _kMouseEventMove, - }); + }, lastMousePos); void tryMoveEdgeOnExit(Offset pos) => handleMouse( { - 'x': pos.dx, - 'y': pos.dy, 'buttons': 0, 'type': _kMouseEventMove, }, + pos, onExit: true, ); @@ -550,17 +541,27 @@ class InputModel { return Offset(x, y); } - void handleMouse( - Map evt, { - bool onExit = false, - }) { - double x = evt['x']; - double y = max(0.0, evt['y']); - final cursorModel = parent.target!.cursorModel; + void handlePointerEvent(String kind, String type, PointerEvent e) { + if (checkPeerControlProtected(e.position.dx, max(0.0, e.position.dy))) { + return; + } + // Only touch events are handled for now. So we can just ignore buttons. + // to-do: handle mouse events + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + modify({ + kind: {type: e.position.toString()} + }) + }), + ); + } + bool checkPeerControlProtected(double x, double y) { + final cursorModel = parent.target!.cursorModel; if (cursorModel.isPeerControlProtected) { lastMousePos = ui.Offset(x, y); - return; + return true; } if (!cursorModel.gotMouseControl) { @@ -571,10 +572,23 @@ class InputModel { cursorModel.gotMouseControl = true; } else { lastMousePos = ui.Offset(x, y); - return; + return true; } } lastMousePos = ui.Offset(x, y); + return false; + } + + void handleMouse( + Map evt, + Offset offset, { + bool onExit = false, + }) { + double x = offset.dx; + double y = max(0.0, offset.dy); + if (checkPeerControlProtected(x, y)) { + return; + } var type = ''; var isMove = false; @@ -615,8 +629,7 @@ class InputModel { kForwardMouseButton: 'forward' }; evt['buttons'] = mapButtons[evt['buttons']] ?? ''; - - bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(evt)); + bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt))); } Point? handlePointerDevicePos( From 933c99110ccc3e607e70f2ba97211266b5fcf5eb Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 07:39:16 +0800 Subject: [PATCH 03/20] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 85 ++++++++++++++++++++--------- 1 file changed, 58 insertions(+), 27 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index e8cd92a9c..ef1a90d6e 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -62,11 +62,11 @@ class InputModel { int _lastButtons = 0; Offset lastMousePos = Offset.zero; - get id => parent.target?.id ?? ""; - late final SessionID sessionId; bool get keyboardPerm => parent.target!.ffiModel.keyboard; + String get id => parent.target?.id ?? ''; + String? get peerPlatform => parent.target?.ffiModel.pi.platform; InputModel(this.parent) { sessionId = parent.target!.sessionId; @@ -334,21 +334,26 @@ class InputModel { void onPointerPanZoomStart(PointerPanZoomStartEvent e) { _lastScale = 1.0; _stopFling = true; - handlePointerEvent('touch', 'pan_start', e); + + if (peerPlatform == kPeerPlatformAndroid) { + handlePointerEvent('touch', 'pan_start', e.position); + } } // https://docs.flutter.dev/release/breaking-changes/trackpad-gestures void onPointerPanZoomUpdate(PointerPanZoomUpdateEvent e) { - final scale = ((e.scale - _lastScale) * 1000).toInt(); - _lastScale = e.scale; + if (peerPlatform != kPeerPlatformAndroid) { + final scale = ((e.scale - _lastScale) * 1000).toInt(); + _lastScale = e.scale; - if (scale != 0) { - bind.sessionSendPointer( - sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': scale} - })); - return; + if (scale != 0) { + bind.sessionSendPointer( + sessionId: sessionId, + msg: json.encode({ + 'touch': {'scale': scale} + })); + return; + } } final delta = e.panDelta; @@ -356,7 +361,8 @@ class InputModel { var x = delta.dx.toInt(); var y = delta.dy.toInt(); - if (parent.target?.ffiModel.pi.platform == kPeerPlatformLinux) { + if (peerPlatform == kPeerPlatformLinux || + peerPlatform == kPeerPlatformAndroid) { _trackpadScrollUnsent += (delta * _trackpadSpeed); x = _trackpadScrollUnsent.dx.truncate(); y = _trackpadScrollUnsent.dy.truncate(); @@ -372,9 +378,13 @@ class InputModel { } } if (x != 0 || y != 0) { - bind.sessionSendMouse( - sessionId: sessionId, - msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); + if (peerPlatform == kPeerPlatformAndroid) { + handlePointerEvent('touch', 'pan_move', e.delta); + } else { + bind.sessionSendMouse( + sessionId: sessionId, + msg: '{"type": "trackpad", "x": "$x", "y": "$y"}'); + } } } @@ -430,6 +440,11 @@ class InputModel { } void onPointerPanZoomEnd(PointerPanZoomEndEvent e) { + if (peerPlatform == kPeerPlatformAndroid) { + handlePointerEvent('touch', 'pan_end', e.position); + return; + } + bind.sessionSendPointer( sessionId: sessionId, msg: json.encode({ @@ -541,23 +556,39 @@ class InputModel { return Offset(x, y); } - void handlePointerEvent(String kind, String type, PointerEvent e) { - if (checkPeerControlProtected(e.position.dx, max(0.0, e.position.dy))) { + void handlePointerEvent(String kind, String type, Offset offset) { + double x = offset.dx; + double y = max(0.0, offset.dy); + if (_checkPeerControlProtected(x, y)) { return; } // Only touch events are handled for now. So we can just ignore buttons. // to-do: handle mouse events - bind.sessionSendPointer( - sessionId: sessionId, - msg: json.encode({ - modify({ - kind: {type: e.position.toString()} - }) - }), + + final isMoveTypes = ['pan', 'pan_start', 'pan_end']; + final pos = handlePointerDevicePos( + x, + y, + isMoveTypes.contains(type), + type, ); + if (pos == null) { + return; + } + + final evt = { + kind: { + type: { + 'x': '${pos.x}', + 'y': '${pos.y}', + } + } + }; + bind.sessionSendPointer( + sessionId: sessionId, msg: json.encode({modify(evt)})); } - bool checkPeerControlProtected(double x, double y) { + bool _checkPeerControlProtected(double x, double y) { final cursorModel = parent.target!.cursorModel; if (cursorModel.isPeerControlProtected) { lastMousePos = ui.Offset(x, y); @@ -586,7 +617,7 @@ class InputModel { }) { double x = offset.dx; double y = max(0.0, offset.dy); - if (checkPeerControlProtected(x, y)) { + if (_checkPeerControlProtected(x, y)) { return; } From d6f1abad959e21e4dbdc4cb05c1a976083b8bb2c Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 09:00:23 +0800 Subject: [PATCH 04/20] tmp commit Signed-off-by: dignow --- .../com/carriez/flutter_hbb/InputService.kt | 31 ++++++++++++++ .../com/carriez/flutter_hbb/MainService.kt | 17 ++++++-- libs/scrap/src/android/ffi.rs | 6 +-- src/server/connection.rs | 42 +++++++++++++++++-- 4 files changed, 85 insertions(+), 11 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 905a2734d..8e14a590d 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -26,6 +26,13 @@ const val WHEEL_BUTTON_UP = 34 const val WHEEL_DOWN = 523331 const val WHEEL_UP = 963 +const val TOUCH_SCALE_START = 1 +const val TOUCH_SCALE = 2 +const val TOUCH_SCALE_END = 3 +const val TOUCH_PAN_START = 4 +const val TOUCH_PAN_UPDATE = 5 +const val TOUCH_PAN_END = 6 + const val WHEEL_STEP = 120 const val WHEEL_DURATION = 50L const val LONG_TAP_DELAY = 200L @@ -167,6 +174,30 @@ class InputService : AccessibilityService() { } } + @RequiresApi(Build.VERSION_CODES.N) + fun onTouchInput(mask: Int, _x: Int, _y: Int) { + val x = max(0, _x) + val y = max(0, _y) + when (mask) { + TOUCH_PAN_UPDATE -> { + mouseX += x * SCREEN_INFO.scale + mouseY += y * SCREEN_INFO.scale + continueGesture(mouseX, mouseY) + } + TOUCH_PAN_START -> { + mouseX = x * SCREEN_INFO.scale + mouseY = y * SCREEN_INFO.scale + startGesture(mouseX, mouseY) + } + TOUCH_PAN_END -> { + mouseX = x * SCREEN_INFO.scale + mouseY = y * SCREEN_INFO.scale + endGesture(mouseX, mouseY) + } + else -> {} + } + } + @RequiresApi(Build.VERSION_CODES.N) private fun consumeWheelActions() { if (isWheelActionsPolling) { diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index 78e4e451e..f32203703 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -71,17 +71,26 @@ class MainService : Service() { @Keep @RequiresApi(Build.VERSION_CODES.N) - fun rustMouseInput(mask: Int, x: Int, y: Int) { + fun rustPointerInput(kind: String, mask: Int, x: Int, y: Int) { // turn on screen with LIFT_DOWN when screen off - if (!powerManager.isInteractive && mask == LIFT_DOWN) { + if (!powerManager.isInteractive && (kind == "touch" || mask == LIFT_DOWN)) { if (wakeLock.isHeld) { - Log.d(logTag,"Turn on Screen, WakeLock release") + Log.d(logTag, "Turn on Screen, WakeLock release") wakeLock.release() } Log.d(logTag,"Turn on Screen") wakeLock.acquire(5000) } else { - InputService.ctx?.onMouseInput(mask,x,y) + when (name) { + "touch" -> { + InputService.ctx?.onTouchInput(mask, x, y) + } + "mouse" -> { + InputService.ctx?.onMouseInput(mask, x, y) + } + else -> { + } + } } } diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 6855fd3f6..3801bbc87 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -154,7 +154,7 @@ pub extern "system" fn Java_com_carriez_flutter_1hbb_MainService_init( } } -pub fn call_main_service_mouse_input(mask: i32, x: i32, y: i32) -> JniResult<()> { +pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), MAIN_SERVICE_CTX.read().unwrap().as_ref(), @@ -162,9 +162,9 @@ pub fn call_main_service_mouse_input(mask: i32, x: i32, y: i32) -> JniResult<()> let mut env = jvm.attach_current_thread_as_daemon()?; env.call_method( ctx, - "rustMouseInput", + "rustPointerInput", "(III)V", - &[JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + &[JValue(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { diff --git a/src/server/connection.rs b/src/server/connection.rs index e32a4c1c3..913c831da 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1546,8 +1546,10 @@ impl Connection { match msg.union { Some(message::Union::MouseEvent(me)) => { #[cfg(any(target_os = "android", target_os = "ios"))] - if let Err(e) = call_main_service_mouse_input(me.mask, me.x, me.y) { - log::debug!("call_main_service_mouse_input fail:{}", e); + if let Err(e) = + call_main_service_pointer_input("mouse".to_string(), me.mask, me.x, me.y) + { + log::debug!("call_main_service_pointer_input fail:{}", e); } #[cfg(not(any(target_os = "android", target_os = "ios")))] if self.peer_keyboard_enabled() { @@ -1559,8 +1561,40 @@ impl Connection { self.input_mouse(me, self.inner.id()); } } - Some(message::Union::PointerDeviceEvent(pde)) => - { + Some(message::Union::PointerDeviceEvent(pde)) => { + #[cfg(any(target_os = "android", target_os = "ios"))] + if let Err(e) = match pde.union { + Some(pointer_device_event::Union::TouchEvent(touch)) => match touch.union { + Some(touch_event::Union::PanStart(pan_start)) => { + call_main_service_pointer_input( + "touch".to_string(), + 4, + pan_start.x, + pan_start.y, + ) + } + Some(touch_event::Union::PanUpdate(pan_start)) => { + call_main_service_pointer_input( + "touch".to_string(), + 5, + pan_start.x, + pan_start.y, + ) + } + Some(touch_event::Union::PanEnd(pan_start)) => { + call_main_service_pointer_input( + "touch".to_string(), + 6, + pan_start.x, + pan_start.y, + ) + } + _ => Ok(()), + }, + _ => Ok(()), + } { + log::debug!("call_main_service_pointer_input fail:{}", e); + } #[cfg(not(any(target_os = "android", target_os = "ios")))] if self.peer_keyboard_enabled() { MOUSE_MOVE_TIME.store(get_time(), Ordering::SeqCst); From 06ee68f836a4e5635ec41fc460afe39d5e851f41 Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 22:26:13 +0800 Subject: [PATCH 05/20] tmp commit Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index ef1a90d6e..18e9d0d2f 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -649,7 +649,7 @@ class InputModel { if (pos == null) { return; } - evt['x'] = '${pos.x}}'; + evt['x'] = '${pos.x}'; evt['y'] = '${pos.y}'; Map mapButtons = { From 93a600a0a8d86cfdb632f15552cb0b887448e0fc Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 9 Aug 2023 23:42:53 +0800 Subject: [PATCH 06/20] tmp commit Signed-off-by: dignow --- flutter/lib/common/widgets/remote_input.dart | 12 +-- flutter/lib/consts.dart | 3 + flutter/lib/models/input_model.dart | 101 ++++++++++++------- src/flutter_ffi.rs | 41 ++++++-- src/server/connection.rs | 12 +-- src/ui_session_interface.rs | 43 ++++++++ 6 files changed, 155 insertions(+), 57 deletions(-) diff --git a/flutter/lib/common/widgets/remote_input.dart b/flutter/lib/common/widgets/remote_input.dart index 35eaf8a7c..b00cd1fb4 100644 --- a/flutter/lib/common/widgets/remote_input.dart +++ b/flutter/lib/common/widgets/remote_input.dart @@ -6,6 +6,7 @@ import 'package:flutter/gestures.dart'; import 'package:flutter_hbb/models/platform_model.dart'; import 'package:flutter_hbb/common.dart'; +import 'package:flutter_hbb/consts.dart'; import 'package:flutter_hbb/models/model.dart'; import 'package:flutter_hbb/models/input_model.dart'; @@ -263,9 +264,9 @@ class _RawTouchGestureDetectorRegionState if (scale != 0) { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': scale} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', scale) + .toJson())); } } else { // mobile @@ -283,9 +284,8 @@ class _RawTouchGestureDetectorRegionState if (isDesktop) { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': 0} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson())); } else { // mobile _scale = 1; diff --git a/flutter/lib/consts.dart b/flutter/lib/consts.dart index 5376196e4..f26e83e01 100644 --- a/flutter/lib/consts.dart +++ b/flutter/lib/consts.dart @@ -54,6 +54,9 @@ const String kTabLabelSettingPage = "Settings"; const String kWindowPrefix = "wm_"; const int kWindowMainId = 0; +const String kPointerEventKindTouch = "touch"; +const String kPointerEventKindMouse = "mouse"; + // the executable name of the portable version const String kEnvPortableExecutable = "RUSTDESK_APPNAME"; diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 18e9d0d2f..9b1ec4437 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -35,6 +35,24 @@ extension ToString on MouseButtons { } } +class PointerEventToRust { + final String kind; + final String type; + final dynamic value; + + PointerEventToRust(this.kind, this.type, this.value); + + Map toJson() { + return { + 'k': kind, + 'v': { + 't': type, + 'v': value, + } + }; + } +} + class InputModel { final WeakReference parent; String keyboardMode = "legacy"; @@ -349,9 +367,9 @@ class InputModel { if (scale != 0) { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': scale} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', scale) + .toJson())); return; } } @@ -379,7 +397,7 @@ class InputModel { } if (x != 0 || y != 0) { if (peerPlatform == kPeerPlatformAndroid) { - handlePointerEvent('touch', 'pan_move', e.delta); + handlePointerEvent('touch', 'pan_update', Offset(x.toDouble(), y.toDouble())); } else { bind.sessionSendMouse( sessionId: sessionId, @@ -447,9 +465,8 @@ class InputModel { bind.sessionSendPointer( sessionId: sessionId, - msg: json.encode({ - 'touch': {'scale': 0} - })); + msg: json.encode( + PointerEventToRust(kPointerEventKindTouch, 'scale', 0).toJson())); waitLastFlingDone(); _stopFling = false; @@ -558,34 +575,40 @@ class InputModel { void handlePointerEvent(String kind, String type, Offset offset) { double x = offset.dx; - double y = max(0.0, offset.dy); + double y = offset.dy; if (_checkPeerControlProtected(x, y)) { return; } // Only touch events are handled for now. So we can just ignore buttons. // to-do: handle mouse events - final isMoveTypes = ['pan', 'pan_start', 'pan_end']; - final pos = handlePointerDevicePos( - x, - y, - isMoveTypes.contains(type), - type, - ); - if (pos == null) { - return; + late final dynamic evtValue; + if (type == 'pan_update') { + evtValue = { + 'x': '${x.toInt()}', + 'y': '${y.toInt()}', + }; + } else { + final isMoveTypes = ['pan_start', 'pan_end']; + final pos = handlePointerDevicePos( + kPointerEventKindTouch, + x, + y, + isMoveTypes.contains(type), + type, + ); + if (pos == null) { + return; + } + evtValue = { + 'x': '${pos.x}', + 'y': '${pos.y}', + }; } - final evt = { - kind: { - type: { - 'x': '${pos.x}', - 'y': '${pos.y}', - } - } - }; + final evt = PointerEventToRust(kind, type, evtValue).toJson(); bind.sessionSendPointer( - sessionId: sessionId, msg: json.encode({modify(evt)})); + sessionId: sessionId, msg: json.encode(modify(evt))); } bool _checkPeerControlProtected(double x, double y) { @@ -639,6 +662,7 @@ class InputModel { evt['type'] = type; final pos = handlePointerDevicePos( + kPointerEventKindMouse, x, y, isMove, @@ -649,8 +673,13 @@ class InputModel { if (pos == null) { return; } - evt['x'] = '${pos.x}'; - evt['y'] = '${pos.y}'; + if (type != '') { + evt['x'] = 0; + evt['y'] = 0; + } else { + evt['x'] = '${pos.x}'; + evt['y'] = '${pos.y}'; + } Map mapButtons = { kPrimaryMouseButton: 'left', @@ -664,6 +693,7 @@ class InputModel { } Point? handlePointerDevicePos( + String kind, double x, double y, bool isMove, @@ -738,18 +768,15 @@ class InputModel { int maxY = (d.y + d.height).toInt() - 1; evtX = trySetNearestRange(evtX, minX, maxX, 5); evtY = trySetNearestRange(evtY, minY, maxY, 5); - if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) { - // If left mouse up, no early return. - if (buttons != kPrimaryMouseButton || evtType != 'up') { - return null; + if (kind == kPointerEventKindMouse) { + if (evtX < minX || evtY < minY || evtX > maxX || evtY > maxY) { + // If left mouse up, no early return. + if (!(buttons == kPrimaryMouseButton && evtType == 'up')) { + return null; + } } } - if (evtType != '') { - evtX = 0; - evtY = 0; - } - return Point(evtX, evtY); } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index ef4c9e2ba..cf5d23397 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1177,14 +1177,39 @@ pub fn session_send_pointer(session_id: SessionID, msg: String) { let ctrl = m.get("ctrl").is_some(); let shift = m.get("shift").is_some(); let command = m.get("command").is_some(); - if let Some(touch_event) = m.get("touch") { - if let Some(scale) = touch_event.get("scale") { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - if let Some(scale) = scale.as_i64() { - session.send_touch_scale(scale as _, alt, ctrl, shift, command); - } - } - } + match (m.get("k"), m.get("v")) { + (Some(k), Some(v)) => match k.as_str() { + Some("touch") => match v.as_str() { + Some("scale") => match v.get("v") { + Some(scale) => { + if let Some(scale) = scale.as_i64() { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + } + None => {} + }, + Some(pan_event) => match (v.get("x"), v.get("y")) { + (Some(x), Some(y)) => { + if let Some(x) = x.as_i64() { + if let Some(y) = y.as_i64() { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) + { + session.send_touch_pan_event( + pan_event, x as _, y as _, alt, ctrl, shift, command, + ); + } + } + } + } + _ => {} + }, + _ => {} + }, + _ => {} + }, + _ => {} } } } diff --git a/src/server/connection.rs b/src/server/connection.rs index 913c831da..972750fc2 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1573,20 +1573,20 @@ impl Connection { pan_start.y, ) } - Some(touch_event::Union::PanUpdate(pan_start)) => { + Some(touch_event::Union::PanUpdate(pan_update)) => { call_main_service_pointer_input( "touch".to_string(), 5, - pan_start.x, - pan_start.y, + pan_update.x, + pan_update.y, ) } - Some(touch_event::Union::PanEnd(pan_start)) => { + Some(touch_event::Union::PanEnd(pan_end)) => { call_main_service_pointer_input( "touch".to_string(), 6, - pan_start.x, - pan_start.y, + pan_end.x, + pan_end.y, ) } _ => Ok(()), diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 7ffcaac6d..23035a550 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -724,6 +724,49 @@ impl Session { send_pointer_device_event(evt, alt, ctrl, shift, command, self); } + pub fn send_touch_pan_event( + &self, + event: &str, + x: i32, + y: i32, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, + ) { + let mut touch_evt = TouchEvent::new(); + match event { + "pan_start" => { + touch_evt.set_pan_start(TouchPanStart { + x, + y, + ..Default::default() + }); + } + "pan_update" => { + touch_evt.set_pan_update(TouchPanUpdate { + x, + y, + ..Default::default() + }); + } + "pan_end" => { + touch_evt.set_pan_end(TouchPanEnd { + x, + y, + ..Default::default() + }); + } + _ => { + log::warn!("unknown touch pan event: {}", event); + return; + } + }; + let mut evt = PointerDeviceEvent::new(); + evt.set_touch_event(touch_evt); + send_pointer_device_event(evt, alt, ctrl, shift, command, self); + } + pub fn send_mouse( &self, mask: i32, From 9e0feb0b64c14c97eccea3905923dbcc7acbaab5 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 01:02:27 +0800 Subject: [PATCH 07/20] tmp debug Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 8 +++---- src/flutter_ffi.rs | 33 ++++++++++++++--------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 9b1ec4437..5f851c54d 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -585,8 +585,8 @@ class InputModel { late final dynamic evtValue; if (type == 'pan_update') { evtValue = { - 'x': '${x.toInt()}', - 'y': '${y.toInt()}', + 'x': x.toInt(), + 'y': y.toInt(), }; } else { final isMoveTypes = ['pan_start', 'pan_end']; @@ -601,8 +601,8 @@ class InputModel { return; } evtValue = { - 'x': '${pos.x}', - 'y': '${pos.y}', + 'x': pos.x, + 'y': pos.y, }; } diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index cf5d23397..c6f8ca774 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1179,30 +1179,29 @@ pub fn session_send_pointer(session_id: SessionID, msg: String) { let command = m.get("command").is_some(); match (m.get("k"), m.get("v")) { (Some(k), Some(v)) => match k.as_str() { - Some("touch") => match v.as_str() { - Some("scale") => match v.get("v") { + Some("touch") => match v.get("t").and_then(|t| t.as_str()) { + Some("scale") => match v.get("v").and_then(|s| s.as_i64()) { Some(scale) => { - if let Some(scale) = scale.as_i64() { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_touch_scale(scale as _, alt, ctrl, shift, command); - } + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); } } None => {} }, - Some(pan_event) => match (v.get("x"), v.get("y")) { - (Some(x), Some(y)) => { - if let Some(x) = x.as_i64() { - if let Some(y) = y.as_i64() { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) - { - session.send_touch_pan_event( - pan_event, x as _, y as _, alt, ctrl, shift, command, - ); - } + Some(pan_event) => match v.get("v") { + Some(v) => match ( + v.get("x").and_then(|x| x.as_i64()), + v.get("y").and_then(|y| y.as_i64()), + ) { + (Some(x), Some(y)) => { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_pan_event( + pan_event, x as _, y as _, alt, ctrl, shift, command, + ); } } - } + _ => {} + }, _ => {} }, _ => {} From da16a799fa6851289bc18b081cfa4e715e6163f3 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 01:10:59 +0800 Subject: [PATCH 08/20] fix build Signed-off-by: dignow --- libs/scrap/src/android/ffi.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 3801bbc87..78908a2ba 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -164,7 +164,7 @@ pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) ctx, "rustPointerInput", "(III)V", - &[JValue(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + &[JValue::String(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { From 9476d7fdbb7edb3b3ebcaa8326622918b71af227 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 01:24:39 +0800 Subject: [PATCH 09/20] try fix build Signed-off-by: dignow --- libs/scrap/src/android/ffi.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index 78908a2ba..ba775dfd3 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -163,8 +163,8 @@ pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) env.call_method( ctx, "rustPointerInput", - "(III)V", - &[JValue::String(kind), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + "(Ljava/lang/String;III)V", + &[JValue::Object(&JObject::from(name)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { From e89ae475f6a4d03ca79d0382e71d1fb9ea816b40 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 03:01:46 +0800 Subject: [PATCH 10/20] fix build Signed-off-by: dignow --- .../main/kotlin/com/carriez/flutter_hbb/MainService.kt | 2 +- libs/scrap/src/android/ffi.rs | 5 +++-- src/server/connection.rs | 10 +++++----- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt index f32203703..535a3f8c3 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt @@ -81,7 +81,7 @@ class MainService : Service() { Log.d(logTag,"Turn on Screen") wakeLock.acquire(5000) } else { - when (name) { + when (kind) { "touch" -> { InputService.ctx?.onTouchInput(mask, x, y) } diff --git a/libs/scrap/src/android/ffi.rs b/libs/scrap/src/android/ffi.rs index ba775dfd3..e9c60ef93 100644 --- a/libs/scrap/src/android/ffi.rs +++ b/libs/scrap/src/android/ffi.rs @@ -154,17 +154,18 @@ pub extern "system" fn Java_com_carriez_flutter_1hbb_MainService_init( } } -pub fn call_main_service_pointer_input(kind: String, mask: i32, x: i32, y: i32) -> JniResult<()> { +pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> { if let (Some(jvm), Some(ctx)) = ( JVM.read().unwrap().as_ref(), MAIN_SERVICE_CTX.read().unwrap().as_ref(), ) { let mut env = jvm.attach_current_thread_as_daemon()?; + let kind = env.new_string(kind)?; env.call_method( ctx, "rustPointerInput", "(Ljava/lang/String;III)V", - &[JValue::Object(&JObject::from(name)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], + &[JValue::Object(&JObject::from(kind)), JValue::Int(mask), JValue::Int(x), JValue::Int(y)], )?; return Ok(()); } else { diff --git a/src/server/connection.rs b/src/server/connection.rs index 972750fc2..f107d15a6 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -39,7 +39,7 @@ use hbb_common::{ tokio_util::codec::{BytesCodec, Framed}, }; #[cfg(any(target_os = "android", target_os = "ios"))] -use scrap::android::call_main_service_mouse_input; +use scrap::android::call_main_service_pointer_input; use serde_json::{json, value::Value}; use sha2::{Digest, Sha256}; #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1547,7 +1547,7 @@ impl Connection { Some(message::Union::MouseEvent(me)) => { #[cfg(any(target_os = "android", target_os = "ios"))] if let Err(e) = - call_main_service_pointer_input("mouse".to_string(), me.mask, me.x, me.y) + call_main_service_pointer_input("mouse", me.mask, me.x, me.y) { log::debug!("call_main_service_pointer_input fail:{}", e); } @@ -1567,7 +1567,7 @@ impl Connection { Some(pointer_device_event::Union::TouchEvent(touch)) => match touch.union { Some(touch_event::Union::PanStart(pan_start)) => { call_main_service_pointer_input( - "touch".to_string(), + "touch", 4, pan_start.x, pan_start.y, @@ -1575,7 +1575,7 @@ impl Connection { } Some(touch_event::Union::PanUpdate(pan_update)) => { call_main_service_pointer_input( - "touch".to_string(), + "touch", 5, pan_update.x, pan_update.y, @@ -1583,7 +1583,7 @@ impl Connection { } Some(touch_event::Union::PanEnd(pan_end)) => { call_main_service_pointer_input( - "touch".to_string(), + "touch", 6, pan_end.x, pan_end.y, From b9c8df70196ca0eeb9f8115729a3775b1cd38600 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 03:55:03 +0800 Subject: [PATCH 11/20] debug Signed-off-by: dignow --- flutter/lib/models/input_model.dart | 7 +++++-- src/server/connection.rs | 11 ++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index 5f851c54d..eaff4ed6a 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -638,6 +638,7 @@ class InputModel { Offset offset, { bool onExit = false, }) { + debugPrint('REMOVE ME ========================= 111'); double x = offset.dx; double y = max(0.0, offset.dy); if (_checkPeerControlProtected(x, y)) { @@ -671,11 +672,12 @@ class InputModel { buttons: evt['buttons'], ); if (pos == null) { + debugPrint('REMOVE ME ========================= 222'); return; } if (type != '') { - evt['x'] = 0; - evt['y'] = 0; + evt['x'] = '0'; + evt['y'] = '0'; } else { evt['x'] = '${pos.x}'; evt['y'] = '${pos.y}'; @@ -689,6 +691,7 @@ class InputModel { kForwardMouseButton: 'forward' }; evt['buttons'] = mapButtons[evt['buttons']] ?? ''; + debugPrint('REMOVE ME ========================= 333 $evt'); bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt))); } diff --git a/src/server/connection.rs b/src/server/connection.rs index f107d15a6..ef8864499 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1546,9 +1546,7 @@ impl Connection { match msg.union { Some(message::Union::MouseEvent(me)) => { #[cfg(any(target_os = "android", target_os = "ios"))] - if let Err(e) = - call_main_service_pointer_input("mouse", me.mask, me.x, me.y) - { + if let Err(e) = call_main_service_pointer_input("mouse", me.mask, me.x, me.y) { log::debug!("call_main_service_pointer_input fail:{}", e); } #[cfg(not(any(target_os = "android", target_os = "ios")))] @@ -1582,12 +1580,7 @@ impl Connection { ) } Some(touch_event::Union::PanEnd(pan_end)) => { - call_main_service_pointer_input( - "touch", - 6, - pan_end.x, - pan_end.y, - ) + call_main_service_pointer_input("touch", 6, pan_end.x, pan_end.y) } _ => Ok(()), }, From be982d95ea548baeee37d805e198b67a2b97da8a Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 08:37:18 +0800 Subject: [PATCH 12/20] tmp build Signed-off-by: dignow --- .../src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 2 ++ flutter/lib/models/input_model.dart | 6 +----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 8e14a590d..84ef31793 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -190,8 +190,10 @@ class InputService : AccessibilityService() { startGesture(mouseX, mouseY) } TOUCH_PAN_END -> { + endGesture(mouseX, mouseY) mouseX = x * SCREEN_INFO.scale mouseY = y * SCREEN_INFO.scale + continueGesture(mouseX, mouseY) endGesture(mouseX, mouseY) } else -> {} diff --git a/flutter/lib/models/input_model.dart b/flutter/lib/models/input_model.dart index eaff4ed6a..971bbb7e5 100644 --- a/flutter/lib/models/input_model.dart +++ b/flutter/lib/models/input_model.dart @@ -379,8 +379,7 @@ class InputModel { var x = delta.dx.toInt(); var y = delta.dy.toInt(); - if (peerPlatform == kPeerPlatformLinux || - peerPlatform == kPeerPlatformAndroid) { + if (peerPlatform == kPeerPlatformLinux) { _trackpadScrollUnsent += (delta * _trackpadSpeed); x = _trackpadScrollUnsent.dx.truncate(); y = _trackpadScrollUnsent.dy.truncate(); @@ -638,7 +637,6 @@ class InputModel { Offset offset, { bool onExit = false, }) { - debugPrint('REMOVE ME ========================= 111'); double x = offset.dx; double y = max(0.0, offset.dy); if (_checkPeerControlProtected(x, y)) { @@ -672,7 +670,6 @@ class InputModel { buttons: evt['buttons'], ); if (pos == null) { - debugPrint('REMOVE ME ========================= 222'); return; } if (type != '') { @@ -691,7 +688,6 @@ class InputModel { kForwardMouseButton: 'forward' }; evt['buttons'] = mapButtons[evt['buttons']] ?? ''; - debugPrint('REMOVE ME ========================= 333 $evt'); bind.sessionSendMouse(sessionId: sessionId, msg: json.encode(modify(evt))); } From 5f7055e28286174440f7a8ee17ff7bec0d600af2 Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 13:57:57 +0800 Subject: [PATCH 13/20] debug Signed-off-by: dignow --- .../src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 84ef31793..65b7931db 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -180,8 +180,8 @@ class InputService : AccessibilityService() { val y = max(0, _y) when (mask) { TOUCH_PAN_UPDATE -> { - mouseX += x * SCREEN_INFO.scale - mouseY += y * SCREEN_INFO.scale + mouseX -= x * SCREEN_INFO.scale + mouseY -= y * SCREEN_INFO.scale continueGesture(mouseX, mouseY) } TOUCH_PAN_START -> { @@ -193,8 +193,6 @@ class InputService : AccessibilityService() { endGesture(mouseX, mouseY) mouseX = x * SCREEN_INFO.scale mouseY = y * SCREEN_INFO.scale - continueGesture(mouseX, mouseY) - endGesture(mouseX, mouseY) } else -> {} } From 072430cef5c0452b4a1ea36b243b1eddc1d0fbdd Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 14:48:00 +0800 Subject: [PATCH 14/20] debug android scroll Signed-off-by: dignow --- .../kotlin/com/carriez/flutter_hbb/InputService.kt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 65b7931db..55c57729e 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -176,23 +176,21 @@ class InputService : AccessibilityService() { @RequiresApi(Build.VERSION_CODES.N) fun onTouchInput(mask: Int, _x: Int, _y: Int) { - val x = max(0, _x) - val y = max(0, _y) when (mask) { TOUCH_PAN_UPDATE -> { - mouseX -= x * SCREEN_INFO.scale - mouseY -= y * SCREEN_INFO.scale + mouseX -= _x * SCREEN_INFO.scale + mouseY -= _y * SCREEN_INFO.scale continueGesture(mouseX, mouseY) } TOUCH_PAN_START -> { - mouseX = x * SCREEN_INFO.scale - mouseY = y * SCREEN_INFO.scale + mouseX = max(0, _x) * SCREEN_INFO.scale + mouseY = max(0, _y) * SCREEN_INFO.scale startGesture(mouseX, mouseY) } TOUCH_PAN_END -> { endGesture(mouseX, mouseY) - mouseX = x * SCREEN_INFO.scale - mouseY = y * SCREEN_INFO.scale + mouseX = max(0, _x) * SCREEN_INFO.scale + mouseY = max(0, _y) * SCREEN_INFO.scale } else -> {} } From 5b2358c97fb40978b30473b034a913c7d6137cce Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 14:49:49 +0800 Subject: [PATCH 15/20] debug android scroll Signed-off-by: dignow --- .../app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt index 55c57729e..203558968 100644 --- a/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt +++ b/flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/InputService.kt @@ -180,6 +180,8 @@ class InputService : AccessibilityService() { TOUCH_PAN_UPDATE -> { mouseX -= _x * SCREEN_INFO.scale mouseY -= _y * SCREEN_INFO.scale + mouseX = max(0, mouseX); + mouseY = max(0, mouseY); continueGesture(mouseX, mouseY) } TOUCH_PAN_START -> { From 6368ab691c47f6292e2156c2f3734fb46f45fa6e Mon Sep 17 00:00:00 2001 From: dignow Date: Thu, 10 Aug 2023 16:08:30 +0800 Subject: [PATCH 16/20] simple refactor, move code from flutter_ffi.rs to flutter.rs Signed-off-by: dignow --- src/flutter.rs | 79 ++++++++++++++++++++++++++++++++++++++++++++++ src/flutter_ffi.rs | 40 +---------------------- 2 files changed, 80 insertions(+), 39 deletions(-) diff --git a/src/flutter.rs b/src/flutter.rs index 52190ce2e..32be6b1c3 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -1130,6 +1130,85 @@ pub fn stop_global_event_stream(app_type: String) { let _ = GLOBAL_EVENT_STREAM.write().unwrap().remove(&app_type); } +#[inline] +fn session_send_touch_scale( + session_id: SessionID, + v: &serde_json::Value, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, +) { + match v.get("v").and_then(|s| s.as_i64()) { + Some(scale) => { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.send_touch_scale(scale as _, alt, ctrl, shift, command); + } + } + None => {} + } +} + +#[inline] +fn session_send_touch_pan( + session_id: SessionID, + v: &serde_json::Value, + pan_event: &str, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, +) { + match v.get("v") { + Some(v) => match ( + v.get("x").and_then(|x| x.as_i64()), + v.get("y").and_then(|y| y.as_i64()), + ) { + (Some(x), Some(y)) => { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session + .send_touch_pan_event(pan_event, x as _, y as _, alt, ctrl, shift, command); + } + } + _ => {} + }, + _ => {} + } +} + +fn session_send_touch_event( + session_id: SessionID, + v: &serde_json::Value, + alt: bool, + ctrl: bool, + shift: bool, + command: bool, +) { + match v.get("t").and_then(|t| t.as_str()) { + Some("scale") => session_send_touch_scale(session_id, v, alt, ctrl, shift, command), + Some(pan_event) => { + session_send_touch_pan(session_id, v, pan_event, alt, ctrl, shift, command) + } + _ => {} + } +} + +pub fn session_send_pointer(session_id: SessionID, msg: String) { + if let Ok(m) = serde_json::from_str::>(&msg) { + let alt = m.get("alt").is_some(); + let ctrl = m.get("ctrl").is_some(); + let shift = m.get("shift").is_some(); + let command = m.get("command").is_some(); + match (m.get("k"), m.get("v")) { + (Some(k), Some(v)) => match k.as_str() { + Some("touch") => session_send_touch_event(session_id, v, alt, ctrl, shift, command), + _ => {} + }, + _ => {} + } + } +} + #[no_mangle] unsafe extern "C" fn get_rgba() {} diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index c6f8ca774..34a1b61e3 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1172,45 +1172,7 @@ pub fn main_clear_ab() { } pub fn session_send_pointer(session_id: SessionID, msg: String) { - if let Ok(m) = serde_json::from_str::>(&msg) { - let alt = m.get("alt").is_some(); - let ctrl = m.get("ctrl").is_some(); - let shift = m.get("shift").is_some(); - let command = m.get("command").is_some(); - match (m.get("k"), m.get("v")) { - (Some(k), Some(v)) => match k.as_str() { - Some("touch") => match v.get("t").and_then(|t| t.as_str()) { - Some("scale") => match v.get("v").and_then(|s| s.as_i64()) { - Some(scale) => { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_touch_scale(scale as _, alt, ctrl, shift, command); - } - } - None => {} - }, - Some(pan_event) => match v.get("v") { - Some(v) => match ( - v.get("x").and_then(|x| x.as_i64()), - v.get("y").and_then(|y| y.as_i64()), - ) { - (Some(x), Some(y)) => { - if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { - session.send_touch_pan_event( - pan_event, x as _, y as _, alt, ctrl, shift, command, - ); - } - } - _ => {} - }, - _ => {} - }, - _ => {} - }, - _ => {} - }, - _ => {} - } - } + super::flutter::session_send_pointer(session_id, msg); } pub fn session_send_mouse(session_id: SessionID, msg: String) { From e205577145d52e2174a7bdc76d12b30123fa5e18 Mon Sep 17 00:00:00 2001 From: dignow Date: Mon, 14 Aug 2023 18:28:31 +0800 Subject: [PATCH 17/20] refact, tab to window, flutter data, init commit Signed-off-by: dignow --- flutter/lib/consts.dart | 2 + .../lib/desktop/pages/remote_tab_page.dart | 34 ++++++++- flutter/lib/models/model.dart | 50 +++++++++++++- flutter/lib/utils/multi_window_manager.dart | 69 ++++++++++++------- 4 files changed, 127 insertions(+), 28 deletions(-) diff --git a/flutter/lib/consts.dart b/flutter/lib/consts.dart index b3ec3aa9d..0621bc807 100644 --- a/flutter/lib/consts.dart +++ b/flutter/lib/consts.dart @@ -5,6 +5,7 @@ import 'package:flutter_hbb/common.dart'; import 'package:flutter_hbb/models/state_model.dart'; const double kDesktopRemoteTabBarHeight = 28.0; +const int kInvalidWindowId = -1; const int kMainWindowId = 0; const String kPeerPlatformWindows = "Windows"; @@ -39,6 +40,7 @@ const String kWindowEventGetSessionIdList = "get_session_id_list"; const String kWindowEventMoveTabToNewWindow = "move_tab_to_new_window"; const String kWindowEventCloseForSeparateWindow = "close_for_separate_window"; +const String kWindowEventSendNewWindowData = "send_new_window_data"; const String kOptionOpenNewConnInTabs = "enable-open-new-connections-in-tabs"; const String kOptionOpenInTabs = "allow-open-in-tabs"; diff --git a/flutter/lib/desktop/pages/remote_tab_page.dart b/flutter/lib/desktop/pages/remote_tab_page.dart index 3762a2b52..01f4e5ca0 100644 --- a/flutter/lib/desktop/pages/remote_tab_page.dart +++ b/flutter/lib/desktop/pages/remote_tab_page.dart @@ -7,6 +7,7 @@ import 'package:flutter/material.dart'; import 'package:flutter_hbb/common.dart'; import 'package:flutter_hbb/common/shared_state.dart'; import 'package:flutter_hbb/consts.dart'; +import 'package:flutter_hbb/models/model.dart'; import 'package:flutter_hbb/models/state_model.dart'; import 'package:flutter_hbb/desktop/pages/remote_page.dart'; import 'package:flutter_hbb/desktop/widgets/remote_toolbar.dart'; @@ -148,9 +149,40 @@ class _ConnectionTabPageState extends State { .toList() .join(';'); } else if (call.method == kWindowEventCloseForSeparateWindow) { - final peerId = call.arguments; + debugPrint('REMOVE ME ============================= ${call.arguments}'); + final peerId = call.arguments['peerId']; + final newWindowId = call.arguments['newWindowId']; + late RemotePage page; + try { + page = tabController.state.value.tabs.firstWhere((tab) { + return tab.key == peerId; + }).page as RemotePage; + } catch (e) { + debugPrint('Failed to find tab for peerId $peerId'); + return false; + } + final sendRes = await rustDeskWinManager.call( + newWindowId, + kWindowEventSendNewWindowData, + page.ffi.ffiModel.cachedPeerData) as bool; + if (!sendRes) { + return false; + } + // Pass the required data to new window. closeSessionOnDispose[peerId] = false; tabController.closeBy(peerId); + return true; + } else if (call.method == kWindowEventSendNewWindowData) { + if (peerId == null) { + return false; + } + if (tabController.state.value.tabs.isEmpty) { + return false; + } + final page = tabController.state.value.tabs[0].page as RemotePage; + page.ffi.ffiModel + .handleCachedPeerData(call.arguments as CachedPeerData, peerId!); + return true; } _update_remote_count(); }); diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 4dc17f9c2..286e89a39 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -41,7 +41,19 @@ final _waitForImageDialogShow = {}; final _waitForFirstImage = {}; final _constSessionId = Uuid().v4obj(); +class CachedPeerData { + Map updatePrivacyMode = {}; + Map peerInfo = {}; + List> cursorDataList = []; + Map lastCursorId = {}; + bool secure = false; + bool direct = false; + + CachedPeerData(); +} + class FfiModel with ChangeNotifier { + CachedPeerData cachedPeerData = CachedPeerData(); PeerInfo _pi = PeerInfo(); Display _display = Display(); @@ -117,6 +129,8 @@ class FfiModel with ChangeNotifier { } setConnectionType(String peerId, bool secure, bool direct) { + cachedPeerData.secure = secure; + cachedPeerData.direct = direct; _secure = secure; _direct = direct; try { @@ -143,6 +157,22 @@ class FfiModel with ChangeNotifier { _permissions.clear(); } + handleCachedPeerData(CachedPeerData data, String peerId) async { + handleMsgBox({ + 'type': 'success', + 'title': 'Successful', + 'text': 'Connected, waiting for image...', + 'link': '', + }, sessionId, peerId); + updatePrivacyMode(data.updatePrivacyMode, sessionId, peerId); + setConnectionType(peerId, data.secure, data.direct); + handlePeerInfo(data.peerInfo, peerId); + for (var element in data.cursorDataList) { + handleCursorData(element); + } + handleCursorId(data.lastCursorId); + } + // todo: why called by two position StreamEventHandler startEventListener(SessionID sessionId, String peerId) { return (evt) async { @@ -159,9 +189,9 @@ class FfiModel with ChangeNotifier { } else if (name == 'switch_display') { handleSwitchDisplay(evt, sessionId, peerId); } else if (name == 'cursor_data') { - await parent.target?.cursorModel.updateCursorData(evt); + await handleCursorData(evt); } else if (name == 'cursor_id') { - await parent.target?.cursorModel.updateCursorId(evt); + await handleCursorId(evt); } else if (name == 'cursor_position') { await parent.target?.cursorModel.updateCursorPosition(evt, peerId); } else if (name == 'clipboard') { @@ -453,6 +483,8 @@ class FfiModel with ChangeNotifier { /// Handle the peer info event based on [evt]. handlePeerInfo(Map evt, String peerId) async { + cachedPeerData.peerInfo = evt; + // recent peer updated by handle_peer_info(ui_session_interface.rs) --> handle_peer_info(client.rs) --> save_config(client.rs) bind.mainLoadRecentPeers(); @@ -568,9 +600,20 @@ class FfiModel with ChangeNotifier { return d; } + handleCursorId(Map evt) async { + cachedPeerData.lastCursorId = evt; + await parent.target?.cursorModel.updateCursorId(evt); + } + + handleCursorData(Map evt) async { + cachedPeerData.cursorDataList.add(evt); + await parent.target?.cursorModel.updateCursorData(evt); + } + /// Handle the peer info synchronization event based on [evt]. handleSyncPeerInfo(Map evt, SessionID sessionId) async { if (evt['displays'] != null) { + cachedPeerData.peerInfo['displays'] = evt['displays']; List displays = json.decode(evt['displays']); List newDisplays = []; for (int i = 0; i < displays.length; ++i) { @@ -1667,7 +1710,8 @@ class FFI { stream.listen((message) { if (closed) return; if (isSessionAdded && !isToNewWindowNotified.value) { - bind.sessionReadyToNewWindow(sessionId: sessionId); + // bind.sessionReadyToNewWindow(sessionId: sessionId); + bind.sessionRefresh(sessionId: sessionId); isToNewWindowNotified.value = true; } () async { diff --git a/flutter/lib/utils/multi_window_manager.dart b/flutter/lib/utils/multi_window_manager.dart index 3f7d995b7..4230182d6 100644 --- a/flutter/lib/utils/multi_window_manager.dart +++ b/flutter/lib/utils/multi_window_manager.dart @@ -28,6 +28,13 @@ extension Index on int { } } +class MultiWindowCallResult { + int windowId; + dynamic result; + + MultiWindowCallResult(this.windowId, this.result); +} + /// Window Manager /// mainly use it in `Main Window` /// use it in sub window is not recommended @@ -49,7 +56,10 @@ class RustDeskMultiWindowManager { 'id': peerId, 'session_id': sessionId, }; - await _newSession( + // It's better to use the window id that returned by _newSession. + // Do not pass original window id to _newSession, + // as this function cann't promise the necessary data is passed to new window. + final multiWindowRes = await _newSession( false, WindowType.RemoteDesktop, kWindowEventNewRemoteDesktop, @@ -57,17 +67,21 @@ class RustDeskMultiWindowManager { _remoteDesktopWindows, jsonEncode(params), ); + // kWindowEventCloseForSeparateWindow will not only close the tab, but also pass the required data to new window. await DesktopMultiWindow.invokeMethod( - windowId, kWindowEventCloseForSeparateWindow, peerId); + windowId, kWindowEventCloseForSeparateWindow, { + 'peerId': peerId, + 'newWindowId': multiWindowRes.windowId, + }); } - newSessionWindow( + Future newSessionWindow( WindowType type, String remoteId, String msg, List windows) async { final windowController = await DesktopMultiWindow.createWindow(msg); + final windowId = windowController.windowId; windowController - ..setFrame(const Offset(0, 0) & - Size(1280 + windowController.windowId * 20, - 720 + windowController.windowId * 20)) + ..setFrame( + const Offset(0, 0) & Size(1280 + windowId * 20, 720 + windowId * 20)) ..center() ..setTitle(getWindowNameWithId( remoteId, @@ -76,11 +90,12 @@ class RustDeskMultiWindowManager { if (Platform.isMacOS) { Future.microtask(() => windowController.show()); } - registerActiveWindow(windowController.windowId); - windows.add(windowController.windowId); + registerActiveWindow(windowId); + windows.add(windowId); + return windowId; } - _newSession( + Future _newSession( bool openInTabs, WindowType type, String methodName, @@ -90,9 +105,10 @@ class RustDeskMultiWindowManager { ) async { if (openInTabs) { if (windows.isEmpty) { - await newSessionWindow(type, remoteId, msg, windows); + final windowId = await newSessionWindow(type, remoteId, msg, windows); + return MultiWindowCallResult(windowId, null); } else { - call(type, methodName, msg); + return call(type, methodName, msg); } } else { if (_inactiveWindows.isNotEmpty) { @@ -103,15 +119,16 @@ class RustDeskMultiWindowManager { await DesktopMultiWindow.invokeMethod(windowId, methodName, msg); WindowController.fromWindowId(windowId).show(); registerActiveWindow(windowId); - return; + return MultiWindowCallResult(windowId, null); } } } - await newSessionWindow(type, remoteId, msg, windows); + final windowId = await newSessionWindow(type, remoteId, msg, windows); + return MultiWindowCallResult(windowId, null); } } - Future newSession( + Future newSession( WindowType type, String methodName, String remoteId, @@ -143,15 +160,15 @@ class RustDeskMultiWindowManager { for (final windowId in windows) { if (await DesktopMultiWindow.invokeMethod( windowId, kWindowEventActiveSession, remoteId)) { - return; + return MultiWindowCallResult(windowId, null); } } } - await _newSession(openInTabs, type, methodName, remoteId, windows, msg); + return _newSession(openInTabs, type, methodName, remoteId, windows, msg); } - Future newRemoteDesktop( + Future newRemoteDesktop( String remoteId, { String? password, String? switchUuid, @@ -168,7 +185,7 @@ class RustDeskMultiWindowManager { ); } - Future newFileTransfer(String remoteId, + Future newFileTransfer(String remoteId, {String? password, bool? forceRelay}) async { return await newSession( WindowType.FileTransfer, @@ -180,7 +197,7 @@ class RustDeskMultiWindowManager { ); } - Future newPortForward(String remoteId, bool isRDP, + Future newPortForward(String remoteId, bool isRDP, {String? password, bool? forceRelay}) async { return await newSession( WindowType.PortForward, @@ -193,18 +210,22 @@ class RustDeskMultiWindowManager { ); } - Future call(WindowType type, String methodName, dynamic args) async { + Future call( + WindowType type, String methodName, dynamic args) async { final wnds = _findWindowsByType(type); if (wnds.isEmpty) { - return; + return MultiWindowCallResult(kInvalidWindowId, null); } for (final windowId in wnds) { if (_activeWindows.contains(windowId)) { - return await DesktopMultiWindow.invokeMethod( - windowId, methodName, args); + final res = + await DesktopMultiWindow.invokeMethod(windowId, methodName, args); + return MultiWindowCallResult(windowId, res); } } - return await DesktopMultiWindow.invokeMethod(wnds[0], methodName, args); + final res = + await DesktopMultiWindow.invokeMethod(wnds[0], methodName, args); + return MultiWindowCallResult(wnds[0], res); } List _findWindowsByType(WindowType type) { From fad88c27189a0cd9f5b93f19240991e66e95cc5f Mon Sep 17 00:00:00 2001 From: dignow Date: Mon, 14 Aug 2023 20:40:58 +0800 Subject: [PATCH 18/20] refact, tab to window, remove rust cache data Signed-off-by: dignow --- flutter/lib/consts.dart | 3 +- .../desktop/pages/desktop_setting_page.dart | 1 - flutter/lib/desktop/pages/remote_page.dart | 5 +- .../lib/desktop/pages/remote_tab_page.dart | 49 +++++---------- .../lib/desktop/widgets/remote_toolbar.dart | 2 +- .../lib/models/desktop_render_texture.dart | 8 +-- flutter/lib/models/model.dart | 63 ++++++++++++++++--- flutter/lib/utils/multi_window_manager.dart | 12 +--- src/client.rs | 2 +- src/client/io_loop.rs | 49 +-------------- src/flutter.rs | 8 ++- src/flutter_ffi.rs | 8 --- src/port_forward.rs | 2 +- src/ui_cm_interface.rs | 1 + src/ui_session_interface.rs | 43 ++----------- 15 files changed, 100 insertions(+), 156 deletions(-) diff --git a/flutter/lib/consts.dart b/flutter/lib/consts.dart index 0621bc807..25c3e16a1 100644 --- a/flutter/lib/consts.dart +++ b/flutter/lib/consts.dart @@ -39,8 +39,7 @@ const String kWindowEventGetRemoteList = "get_remote_list"; const String kWindowEventGetSessionIdList = "get_session_id_list"; const String kWindowEventMoveTabToNewWindow = "move_tab_to_new_window"; -const String kWindowEventCloseForSeparateWindow = "close_for_separate_window"; -const String kWindowEventSendNewWindowData = "send_new_window_data"; +const String kWindowEventGetCachedSessionData = "get_cached_session_data"; const String kOptionOpenNewConnInTabs = "enable-open-new-connections-in-tabs"; const String kOptionOpenInTabs = "allow-open-in-tabs"; diff --git a/flutter/lib/desktop/pages/desktop_setting_page.dart b/flutter/lib/desktop/pages/desktop_setting_page.dart index b868042a4..8d5ddb7dc 100644 --- a/flutter/lib/desktop/pages/desktop_setting_page.dart +++ b/flutter/lib/desktop/pages/desktop_setting_page.dart @@ -17,7 +17,6 @@ import 'package:provider/provider.dart'; import 'package:url_launcher/url_launcher.dart'; import 'package:url_launcher/url_launcher_string.dart'; import 'package:flutter_hbb/desktop/widgets/scroll_wrapper.dart'; -import 'package:window_manager/window_manager.dart'; import '../../common/widgets/dialog.dart'; import '../../common/widgets/login.dart'; diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index 28212e4ca..2daffaccf 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -35,6 +35,7 @@ class RemotePage extends StatefulWidget { Key? key, required this.id, required this.sessionId, + required this.tabWindowId, required this.password, required this.toolbarState, required this.tabController, @@ -44,6 +45,7 @@ class RemotePage extends StatefulWidget { final String id; final SessionID? sessionId; + final int? tabWindowId; final String? password; final ToolbarState toolbarState; final String? switchUuid; @@ -106,6 +108,7 @@ class _RemotePageState extends State password: widget.password, switchUuid: widget.switchUuid, forceRelay: widget.forceRelay, + tabWindowId: widget.tabWindowId, ); WidgetsBinding.instance.addPostFrameCallback((_) { SystemChrome.setEnabledSystemUIMode(SystemUiMode.manual, overlays: []); @@ -204,7 +207,7 @@ class _RemotePageState extends State // https://github.com/flutter/flutter/issues/64935 super.dispose(); debugPrint("REMOTE PAGE dispose session $sessionId ${widget.id}"); - await _renderTexture.destroy(); + await _renderTexture.destroy(widget.tabWindowId != null); // ensure we leave this session, this is a double check bind.sessionEnterOrLeave(sessionId: sessionId, enter: false); DesktopMultiWindow.removeListener(this); diff --git a/flutter/lib/desktop/pages/remote_tab_page.dart b/flutter/lib/desktop/pages/remote_tab_page.dart index 01f4e5ca0..51bcec51d 100644 --- a/flutter/lib/desktop/pages/remote_tab_page.dart +++ b/flutter/lib/desktop/pages/remote_tab_page.dart @@ -7,7 +7,6 @@ import 'package:flutter/material.dart'; import 'package:flutter_hbb/common.dart'; import 'package:flutter_hbb/common/shared_state.dart'; import 'package:flutter_hbb/consts.dart'; -import 'package:flutter_hbb/models/model.dart'; import 'package:flutter_hbb/models/state_model.dart'; import 'package:flutter_hbb/desktop/pages/remote_page.dart'; import 'package:flutter_hbb/desktop/widgets/remote_toolbar.dart'; @@ -56,6 +55,7 @@ class _ConnectionTabPageState extends State { RemoteCountState.init(); peerId = params['id']; final sessionId = params['session_id']; + final tabWindowId = params['tab_window_id']; if (peerId != null) { ConnectionTypeState.init(peerId!); tabController.onSelected = (id) { @@ -78,6 +78,7 @@ class _ConnectionTabPageState extends State { key: ValueKey(peerId), id: peerId!, sessionId: sessionId == null ? null : SessionID(sessionId), + tabWindowId: tabWindowId, password: params['password'], toolbarState: _toolbarState, tabController: tabController, @@ -99,12 +100,14 @@ class _ConnectionTabPageState extends State { print( "[Remote Page] call ${call.method} with args ${call.arguments} from window $fromWindowId"); + dynamic returnValue; // for simplify, just replace connectionId if (call.method == kWindowEventNewRemoteDesktop) { final args = jsonDecode(call.arguments); final id = args['id']; final switchUuid = args['switch_uuid']; final sessionId = args['session_id']; + final tabWindowId = args['tab_window_id']; windowOnTop(windowId()); ConnectionTypeState.init(id); _toolbarState.setShow( @@ -119,6 +122,7 @@ class _ConnectionTabPageState extends State { key: ValueKey(id), id: id, sessionId: sessionId == null ? null : SessionID(sessionId), + tabWindowId: tabWindowId, password: args['password'], toolbarState: _toolbarState, tabController: tabController, @@ -148,43 +152,24 @@ class _ConnectionTabPageState extends State { .map((e) => '${e.key},${(e.page as RemotePage).ffi.sessionId}') .toList() .join(';'); - } else if (call.method == kWindowEventCloseForSeparateWindow) { - debugPrint('REMOVE ME ============================= ${call.arguments}'); - final peerId = call.arguments['peerId']; - final newWindowId = call.arguments['newWindowId']; - late RemotePage page; + } else if (call.method == kWindowEventGetCachedSessionData) { + // Ready to show new window and close old tab. + final peerId = call.arguments; try { - page = tabController.state.value.tabs.firstWhere((tab) { - return tab.key == peerId; - }).page as RemotePage; + final remotePage = tabController.state.value.tabs + .firstWhere((tab) => tab.key == peerId) + .page as RemotePage; + returnValue = remotePage.ffi.ffiModel.cachedPeerData.toString(); } catch (e) { - debugPrint('Failed to find tab for peerId $peerId'); - return false; + debugPrint('Failed to get cached session data: $e'); } - final sendRes = await rustDeskWinManager.call( - newWindowId, - kWindowEventSendNewWindowData, - page.ffi.ffiModel.cachedPeerData) as bool; - if (!sendRes) { - return false; + if (returnValue != null) { + closeSessionOnDispose[peerId] = false; + tabController.closeBy(peerId); } - // Pass the required data to new window. - closeSessionOnDispose[peerId] = false; - tabController.closeBy(peerId); - return true; - } else if (call.method == kWindowEventSendNewWindowData) { - if (peerId == null) { - return false; - } - if (tabController.state.value.tabs.isEmpty) { - return false; - } - final page = tabController.state.value.tabs[0].page as RemotePage; - page.ffi.ffiModel - .handleCachedPeerData(call.arguments as CachedPeerData, peerId!); - return true; } _update_remote_count(); + return returnValue; }); Future.delayed(Duration.zero, () { restoreWindowPosition( diff --git a/flutter/lib/desktop/widgets/remote_toolbar.dart b/flutter/lib/desktop/widgets/remote_toolbar.dart index 6a72fc3a1..9d6dec496 100644 --- a/flutter/lib/desktop/widgets/remote_toolbar.dart +++ b/flutter/lib/desktop/widgets/remote_toolbar.dart @@ -771,7 +771,7 @@ class ScreenAdjustor { updateScreen() async { final v = await rustDeskWinManager.call( WindowType.Main, kWindowGetWindowInfo, ''); - final String valueStr = v; + final String valueStr = v.result; if (valueStr.isEmpty) { _screen = null; } else { diff --git a/flutter/lib/models/desktop_render_texture.dart b/flutter/lib/models/desktop_render_texture.dart index f59373623..f8456e339 100644 --- a/flutter/lib/models/desktop_render_texture.dart +++ b/flutter/lib/models/desktop_render_texture.dart @@ -1,4 +1,3 @@ -import 'package:flutter/material.dart'; import 'package:get/get.dart'; import 'package:texture_rgba_renderer/texture_rgba_renderer.dart'; @@ -21,7 +20,6 @@ class RenderTexture { _sessionId = sessionId; textureRenderer.createTexture(_textureKey).then((id) async { - debugPrint("id: $id, texture_key: $_textureKey"); if (id != -1) { final ptr = await textureRenderer.getTexturePtr(_textureKey); platformFFI.registerTexture(sessionId, ptr); @@ -31,9 +29,11 @@ class RenderTexture { } } - destroy() async { + destroy(bool unregisterTexture) async { if (useTextureRender && _textureKey != -1 && _sessionId != null) { - platformFFI.registerTexture(_sessionId!, 0); + if (unregisterTexture) { + platformFFI.registerTexture(_sessionId!, 0); + } await textureRenderer.closeTexture(_textureKey); _textureKey = -1; } diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 286e89a39..4d0e30e3e 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -4,6 +4,7 @@ import 'dart:io'; import 'dart:math'; import 'dart:ui' as ui; +import 'package:desktop_multi_window/desktop_multi_window.dart'; import 'package:flutter/material.dart'; import 'package:flutter/services.dart'; import 'package:flutter_hbb/consts.dart'; @@ -50,6 +51,37 @@ class CachedPeerData { bool direct = false; CachedPeerData(); + + @override + String toString() { + return jsonEncode({ + 'updatePrivacyMode': updatePrivacyMode, + 'peerInfo': peerInfo, + 'cursorDataList': cursorDataList, + 'lastCursorId': lastCursorId, + 'secure': secure, + 'direct': direct, + }); + } + + static CachedPeerData? fromString(String s) { + try { + final map = jsonDecode(s); + final data = CachedPeerData(); + data.updatePrivacyMode = map['updatePrivacyMode']; + data.peerInfo = map['peerInfo']; + for (final cursorData in map['cursorDataList']) { + data.cursorDataList.add(cursorData); + } + data.lastCursorId = map['lastCursorId']; + data.secure = map['secure']; + data.direct = map['direct']; + return data; + } catch (e) { + debugPrint('Failed to parse CachedPeerData: $e'); + return null; + } + } } class FfiModel with ChangeNotifier { @@ -1628,7 +1660,6 @@ class FFI { /// dialogManager use late to ensure init after main page binding [globalKey] late final dialogManager = OverlayDialogManager(); - late final bool isSessionAdded; late final SessionID sessionId; late final ImageModel imageModel; // session late final FfiModel ffiModel; // session @@ -1647,7 +1678,6 @@ class FFI { late final ElevationModel elevationModel; // session FFI(SessionID? sId) { - isSessionAdded = sId != null; sessionId = sId ?? (isDesktop ? Uuid().v4obj() : _constSessionId); imageModel = ImageModel(WeakReference(this)); ffiModel = FfiModel(WeakReference(this)); @@ -1673,7 +1703,8 @@ class FFI { bool isRdp = false, String? switchUuid, String? password, - bool? forceRelay}) { + bool? forceRelay, + int? tabWindowId}) { closed = false; auditNote = ''; assert(!(isFileTransfer && isPortForward), 'more than one connect type'); @@ -1688,7 +1719,9 @@ class FFI { imageModel.id = id; cursorModel.id = id; } - if (!isSessionAdded) { + // If tabWindowId != null, this session is a "tab -> window" one. + // Else this session is a new one. + if (tabWindowId == null) { // ignore: unused_local_variable final addRes = bind.sessionAddSync( sessionId: sessionId, @@ -1709,9 +1742,25 @@ class FFI { // Preserved for the rgba data. stream.listen((message) { if (closed) return; - if (isSessionAdded && !isToNewWindowNotified.value) { - // bind.sessionReadyToNewWindow(sessionId: sessionId); - bind.sessionRefresh(sessionId: sessionId); + if (tabWindowId != null && !isToNewWindowNotified.value) { + // Session is read to be moved to a new window. + // Get the cached data and handle the cached data. + Future.delayed(Duration.zero, () async { + final cachedData = await DesktopMultiWindow.invokeMethod( + tabWindowId, kWindowEventGetCachedSessionData, id); + if (cachedData == null) { + // unreachable + debugPrint('Unreachable, the cached data is empty.'); + return; + } + final data = CachedPeerData.fromString(cachedData); + if (data == null) { + debugPrint('Unreachable, the cached data cannot be decoded.'); + return; + } + ffiModel.handleCachedPeerData(data, id); + await bind.sessionRefresh(sessionId: sessionId); + }); isToNewWindowNotified.value = true; } () async { diff --git a/flutter/lib/utils/multi_window_manager.dart b/flutter/lib/utils/multi_window_manager.dart index 4230182d6..a8be78c74 100644 --- a/flutter/lib/utils/multi_window_manager.dart +++ b/flutter/lib/utils/multi_window_manager.dart @@ -54,12 +54,10 @@ class RustDeskMultiWindowManager { var params = { 'type': WindowType.RemoteDesktop.index, 'id': peerId, + 'tab_window_id': windowId, 'session_id': sessionId, }; - // It's better to use the window id that returned by _newSession. - // Do not pass original window id to _newSession, - // as this function cann't promise the necessary data is passed to new window. - final multiWindowRes = await _newSession( + await _newSession( false, WindowType.RemoteDesktop, kWindowEventNewRemoteDesktop, @@ -67,12 +65,6 @@ class RustDeskMultiWindowManager { _remoteDesktopWindows, jsonEncode(params), ); - // kWindowEventCloseForSeparateWindow will not only close the tab, but also pass the required data to new window. - await DesktopMultiWindow.invokeMethod( - windowId, kWindowEventCloseForSeparateWindow, { - 'peerId': peerId, - 'newWindowId': multiWindowRes.windowId, - }); } Future newSessionWindow( diff --git a/src/client.rs b/src/client.rs index 84f338c63..798550467 100644 --- a/src/client.rs +++ b/src/client.rs @@ -2366,7 +2366,7 @@ pub trait Interface: Send + Clone + 'static + Sized { fn send(&self, data: Data); fn msgbox(&self, msgtype: &str, title: &str, text: &str, link: &str); fn handle_login_error(&mut self, err: &str) -> bool; - fn handle_peer_info(&mut self, pi: PeerInfo, is_cached_pi: bool); + fn handle_peer_info(&mut self, pi: PeerInfo); fn on_error(&self, err: &str) { self.msgbox("error", "Error", err, ""); } diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index 9e78b17a5..aaf426e28 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -125,18 +125,7 @@ impl Remote { .await { Ok((mut peer, direct, pk)) => { - let is_secured = peer.is_secured(); - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - self.handler - .cache_flutter - .write() - .unwrap() - .is_secured_direct - .replace((is_secured, direct)); - } - self.handler.set_connection_type(is_secured, direct); // flutter -> connection_ready + self.handler.set_connection_type(peer.is_secured(), direct); // flutter -> connection_ready self.handler.update_direct(Some(direct)); if conn_type == ConnType::DEFAULT_CONN { self.handler @@ -1021,12 +1010,7 @@ impl Remote { } } Some(login_response::Union::PeerInfo(pi)) => { - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - self.handler.cache_flutter.write().unwrap().pi = pi.clone(); - } - self.handler.handle_peer_info(pi, false); + self.handler.handle_peer_info(pi); #[cfg(not(feature = "flutter"))] self.check_clipboard_file_context(); if !(self.handler.is_file_transfer() || self.handler.is_port_forward()) { @@ -1073,22 +1057,9 @@ impl Remote { _ => {} }, Some(message::Union::CursorData(cd)) => { - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - let mut lock = self.handler.cache_flutter.write().unwrap(); - if !lock.cursor_data.contains_key(&cd.id) { - lock.cursor_data.insert(cd.id, cd.clone()); - } - } self.handler.set_cursor_data(cd); } Some(message::Union::CursorId(id)) => { - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - self.handler.cache_flutter.write().unwrap().cursor_id = id; - } self.handler.set_cursor_id(id.to_string()); } Some(message::Union::CursorPosition(cp)) => { @@ -1305,16 +1276,6 @@ impl Remote { } } Some(misc::Union::SwitchDisplay(s)) => { - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - self.handler - .cache_flutter - .write() - .unwrap() - .sp - .replace(s.clone()); - } self.handler.handle_peer_switch_display(&s); self.video_sender.send(MediaData::Reset).ok(); if s.width > 0 && s.height > 0 { @@ -1506,12 +1467,6 @@ impl Remote { } } Some(message::Union::PeerInfo(pi)) => { - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - self.handler.cache_flutter.write().unwrap().pi.displays = - pi.displays.clone(); - } self.handler.set_displays(&pi.displays); } _ => {} diff --git a/src/flutter.rs b/src/flutter.rs index 52190ce2e..1feb6b118 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -36,9 +36,11 @@ pub(crate) const APP_TYPE_CM: &str = "cm"; #[cfg(any(target_os = "android", target_os = "ios"))] pub(crate) const APP_TYPE_CM: &str = "main"; -pub(crate) const APP_TYPE_DESKTOP_REMOTE: &str = "remote"; -pub(crate) const APP_TYPE_DESKTOP_FILE_TRANSFER: &str = "file transfer"; -pub(crate) const APP_TYPE_DESKTOP_PORT_FORWARD: &str = "port forward"; +// Do not remove the following constants. +// Uncomment them when they are used. +// pub(crate) const APP_TYPE_DESKTOP_REMOTE: &str = "remote"; +// pub(crate) const APP_TYPE_DESKTOP_FILE_TRANSFER: &str = "file transfer"; +// pub(crate) const APP_TYPE_DESKTOP_PORT_FORWARD: &str = "port forward"; lazy_static::lazy_static! { pub(crate) static ref CUR_SESSION_ID: RwLock = Default::default(); diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index ab7acc104..c36d56665 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -597,14 +597,6 @@ pub fn session_change_resolution(session_id: SessionID, display: i32, width: i32 } } -pub fn session_ready_to_new_window(session_id: SessionID) { - #[cfg(not(any(target_os = "android", target_os = "ios")))] - if let Some(session) = SESSIONS.write().unwrap().get_mut(&session_id) { - session.restore_flutter_cache(); - session.refresh_video(); - } -} - pub fn session_set_size(_session_id: SessionID, _width: usize, _height: usize) { #[cfg(feature = "flutter_texture_render")] if let Some(session) = SESSIONS.write().unwrap().get_mut(&_session_id) { diff --git a/src/port_forward.rs b/src/port_forward.rs index 1e918cce1..6a087abe2 100644 --- a/src/port_forward.rs +++ b/src/port_forward.rs @@ -146,7 +146,7 @@ async fn connect_and_login( return Ok(None); } Some(login_response::Union::PeerInfo(pi)) => { - interface.handle_peer_info(pi, false); + interface.handle_peer_info(pi); break; } _ => {} diff --git a/src/ui_cm_interface.rs b/src/ui_cm_interface.rs index 29828d6b7..16fa59631 100644 --- a/src/ui_cm_interface.rs +++ b/src/ui_cm_interface.rs @@ -325,6 +325,7 @@ impl IpcTaskRunner { // for tmp use, without real conn id let mut write_jobs: Vec = Vec::new(); + #[cfg(windows)] let is_authorized = self.cm.is_authorized(self.conn_id); #[cfg(windows)] diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 1fdff8144..eb902e3a9 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -48,17 +48,6 @@ pub static IS_IN: AtomicBool = AtomicBool::new(false); const CHANGE_RESOLUTION_VALID_TIMEOUT_SECS: u64 = 15; -#[cfg(feature = "flutter")] -#[cfg(not(any(target_os = "android", target_os = "ios")))] -#[derive(Default)] -pub struct CacheFlutter { - pub pi: PeerInfo, - pub sp: Option, - pub cursor_data: HashMap, - pub cursor_id: u64, - pub is_secured_direct: Option<(bool, bool)>, -} - #[derive(Clone, Default)] pub struct Session { pub session_id: SessionID, // different from the one in LoginConfigHandler, used for flutter UI message pass @@ -73,9 +62,6 @@ pub struct Session { pub server_file_transfer_enabled: Arc>, pub server_clipboard_enabled: Arc>, pub last_change_display: Arc>, - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - pub cache_flutter: Arc>, } #[derive(Clone)] @@ -1095,7 +1081,7 @@ impl Interface for Session { handle_login_error(self.lc.clone(), err, self) } - fn handle_peer_info(&mut self, mut pi: PeerInfo, is_cached_pi: bool) { + fn handle_peer_info(&mut self, mut pi: PeerInfo) { log::debug!("handle_peer_info :{:?}", pi); pi.username = self.lc.read().unwrap().get_username(&pi); if pi.current_display as usize >= pi.displays.len() { @@ -1116,12 +1102,10 @@ impl Interface for Session { self.msgbox("error", "Remote Error", "No Display", ""); return; } - if !is_cached_pi { - self.try_change_init_resolution(pi.current_display); - let p = self.lc.read().unwrap().should_auto_login(); - if !p.is_empty() { - input_os_password(p, true, self.clone()); - } + self.try_change_init_resolution(pi.current_display); + let p = self.lc.read().unwrap().should_auto_login(); + if !p.is_empty() { + input_os_password(p, true, self.clone()); } let current = &pi.displays[pi.current_display as usize]; self.set_display( @@ -1222,23 +1206,6 @@ impl Session { pub fn ctrl_alt_del(&self) { self.send_key_event(&crate::keyboard::client::event_ctrl_alt_del()); } - - #[cfg(feature = "flutter")] - #[cfg(not(any(target_os = "android", target_os = "ios")))] - pub fn restore_flutter_cache(&mut self) { - if let Some((is_secured, direct)) = self.cache_flutter.read().unwrap().is_secured_direct { - self.set_connection_type(is_secured, direct); - } - let pi = self.cache_flutter.read().unwrap().pi.clone(); - self.handle_peer_info(pi, true); - if let Some(sp) = self.cache_flutter.read().unwrap().sp.as_ref() { - self.handle_peer_switch_display(sp); - } - for (_, cd) in self.cache_flutter.read().unwrap().cursor_data.iter() { - self.set_cursor_data(cd.clone()); - } - self.set_cursor_id(self.cache_flutter.read().unwrap().cursor_id.to_string()); - } } #[tokio::main(flavor = "current_thread")] From 9adac5686be4914134e124162694b38338c66c2a Mon Sep 17 00:00:00 2001 From: dignow Date: Wed, 16 Aug 2023 21:52:03 +0800 Subject: [PATCH 19/20] fix unregister texture condition Signed-off-by: dignow --- flutter/lib/desktop/pages/remote_page.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index 2daffaccf..0d51cabdd 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -207,7 +207,7 @@ class _RemotePageState extends State // https://github.com/flutter/flutter/issues/64935 super.dispose(); debugPrint("REMOTE PAGE dispose session $sessionId ${widget.id}"); - await _renderTexture.destroy(widget.tabWindowId != null); + await _renderTexture.destroy(closeSession); // ensure we leave this session, this is a double check bind.sessionEnterOrLeave(sessionId: sessionId, enter: false); DesktopMultiWindow.removeListener(this); From da9fb46b6a65d9c96d15ac4c18cac8fa5be59a5e Mon Sep 17 00:00:00 2001 From: 21pages Date: Wed, 23 Aug 2023 12:20:19 +0800 Subject: [PATCH 20/20] fix pushAb Signed-off-by: 21pages --- flutter/lib/models/ab_model.dart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flutter/lib/models/ab_model.dart b/flutter/lib/models/ab_model.dart index e91e42ef9..cbb7f7471 100644 --- a/flutter/lib/models/ab_model.dart +++ b/flutter/lib/models/ab_model.dart @@ -225,7 +225,7 @@ class AbModel { final api = "${await bind.mainGetApiServer()}/api/ab"; var authHeaders = getHttpHeaders(); authHeaders['Content-Type'] = "application/json"; - final body = jsonEncode(_serialize()); + final body = jsonEncode({"data": jsonEncode(_serialize())}); http.Response resp; // support compression if (licensedDevices > 0 && body.length > 1024) {