android floating window (#8268)

Signed-off-by: 21pages <sunboeasy@gmail.com>

flutter/android/app/build.gradle

@@ -106,5 +106,6 @@ dependencies {
     implementation "androidx.media:media:1.6.0"
     implementation 'com.github.getActivity:XXPermissions:18.5'
     implementation("org.jetbrains.kotlin:kotlin-stdlib") { version { strictly("$kotlin_version") } }
+    implementation 'com.caverock:androidsvg-aar:1.4'
 }
 

flutter/android/app/src/main/AndroidManifest.xml

@@ -81,6 +81,11 @@
             android:name=".MainService"
             android:enabled="true"
             android:foregroundServiceType="mediaProjection" />
+
+        <service
+            android:name=".FloatingWindowService"
+            android:enabled="true" />
+
         <!--
  Don't delete the meta-data below.
              This is used by the Flutter tool to generate GeneratedPluginRegistrant.java

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/FloatingWindowService.kt

@@ -0,0 +1,307 @@
+package com.carriez.flutter_hbb
+
+import android.annotation.SuppressLint
+import android.app.PendingIntent
+import android.app.Service
+import android.content.Intent
+import android.content.res.Configuration
+import android.graphics.Bitmap
+import android.graphics.Canvas
+import android.graphics.PixelFormat
+import android.graphics.drawable.BitmapDrawable
+import android.graphics.drawable.Drawable
+import android.os.Build
+import android.os.IBinder
+import android.util.Log
+import android.view.Gravity
+import android.view.MotionEvent
+import android.view.View
+import android.view.WindowManager
+import android.view.WindowManager.LayoutParams.FLAG_LAYOUT_IN_SCREEN
+import android.view.WindowManager.LayoutParams.FLAG_NOT_FOCUSABLE
+import android.view.WindowManager.LayoutParams.FLAG_NOT_TOUCH_MODAL
+import android.widget.ImageView
+import android.widget.PopupMenu
+import com.caverock.androidsvg.SVG
+import ffi.FFI
+import kotlin.math.abs
+
+class FloatingWindowService : Service(), View.OnTouchListener {
+
+    private lateinit var windowManager: WindowManager
+    private lateinit var layoutParams: WindowManager.LayoutParams
+    private lateinit var floatingView: ImageView
+    private lateinit var originalDrawable: Drawable
+    private lateinit var leftHalfDrawable: Drawable
+    private lateinit var rightHalfDrawable: Drawable
+
+    private var dragging = false
+    private var lastDownX = 0f
+    private var lastDownY = 0f
+
+    companion object {
+        private val logTag = "floatingService"
+        private var firsCreate = true
+        private var viewWidth = 120
+        private var viewHeight = 120
+        private const val MIN_VIEW_SIZE = 32 // size 0 does not help prevent the service from being killed
+        private const val MAX_VIEW_SIZE = 320
+        private var viewTransparency = 1f // 0 means invisible but can help prevent the service from being killed
+        private var customSvg = ""
+        private var lastLayoutX = 0
+        private var lastLayoutY = 0
+    }
+
+    override fun onBind(intent: Intent): IBinder? {
+        return null
+    }
+
+    override fun onCreate() {
+        super.onCreate()
+        windowManager = getSystemService(WINDOW_SERVICE) as WindowManager
+        try {
+            if (firsCreate) {
+                firsCreate = false
+                onFirstCreate(windowManager)
+            }
+            Log.d(logTag, "floating window size: $viewWidth x $viewHeight, transparency: $viewTransparency, lastLayoutX: $lastLayoutX, lastLayoutY: $lastLayoutY, customSvg: $customSvg")
+            createView(windowManager)
+            Log.d(logTag, "onCreate success")
+        } catch (e: Exception) {
+            Log.d(logTag, "onCreate failed: $e")
+        }
+    }
+
+    @SuppressLint("ClickableViewAccessibility")
+    private fun createView(windowManager: WindowManager) {
+        floatingView = ImageView(this)
+        originalDrawable = resources.getDrawable(R.drawable.floating_window, null)
+        if (customSvg.isNotEmpty()) {
+            try {
+                val svg = SVG.getFromString(customSvg)
+                Log.d(logTag, "custom svg info: ${svg.documentWidth} x ${svg.documentHeight}");
+                // This make the svg render clear
+               svg.documentWidth = viewWidth * 1f
+               svg.documentHeight = viewHeight * 1f
+                originalDrawable = svg.renderToPicture().let {
+                    BitmapDrawable(
+                        resources,
+                        Bitmap.createBitmap(it.width, it.height, Bitmap.Config.ARGB_8888)
+                            .also { bitmap ->
+                                it.draw(Canvas(bitmap))
+                            })
+                }
+                floatingView.setLayerType(View.LAYER_TYPE_SOFTWARE, null);
+                Log.d(logTag, "custom svg loaded")
+            } catch (e: Exception) {
+                e.printStackTrace()
+            }
+        }
+        val originalBitmap = Bitmap.createBitmap(
+            originalDrawable.intrinsicWidth,
+            originalDrawable.intrinsicHeight,
+            Bitmap.Config.ARGB_8888
+        )
+        val canvas = Canvas(originalBitmap)
+        originalDrawable.setBounds(
+            0,
+            0,
+            originalDrawable.intrinsicWidth,
+            originalDrawable.intrinsicHeight
+        )
+        originalDrawable.draw(canvas)
+        val leftHalfBitmap = Bitmap.createBitmap(
+            originalBitmap,
+            0,
+            0,
+            originalDrawable.intrinsicWidth / 2,
+            originalDrawable.intrinsicHeight
+        )
+        val rightHalfBitmap = Bitmap.createBitmap(
+            originalBitmap,
+            originalDrawable.intrinsicWidth / 2,
+            0,
+            originalDrawable.intrinsicWidth / 2,
+            originalDrawable.intrinsicHeight
+        )
+        leftHalfDrawable = BitmapDrawable(resources, leftHalfBitmap)
+        rightHalfDrawable = BitmapDrawable(resources, rightHalfBitmap)
+
+        floatingView.setImageDrawable(rightHalfDrawable)
+        floatingView.setOnTouchListener(this)
+        floatingView.alpha = viewTransparency * 1f
+
+        val flags = FLAG_LAYOUT_IN_SCREEN or FLAG_NOT_TOUCH_MODAL or FLAG_NOT_FOCUSABLE
+        layoutParams = WindowManager.LayoutParams(
+            viewWidth / 2,
+            viewHeight,
+            if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) WindowManager.LayoutParams.TYPE_APPLICATION_OVERLAY else WindowManager.LayoutParams.TYPE_PHONE,
+            flags,
+            PixelFormat.TRANSLUCENT
+        )
+
+        layoutParams.gravity = Gravity.TOP or Gravity.START
+        layoutParams.x = lastLayoutX
+        layoutParams.y = lastLayoutY
+
+        windowManager.addView(floatingView, layoutParams)
+        moveToScreenSide()
+    }
+
+    private fun onFirstCreate(windowManager: WindowManager) {
+        val wh = getScreenSize(windowManager)
+        val w = wh.first
+        val h = wh.second
+        // size
+        FFI.getLocalOption("floating-window-size").let {
+            if (it.isNotEmpty()) {
+                try {
+                    val size = it.toInt()
+                    if (size in MIN_VIEW_SIZE..MAX_VIEW_SIZE && size <= w / 2 && size <= h / 2) {
+                        viewWidth = size
+                        viewHeight = size
+                    }
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                }
+            }
+        }
+        // transparency
+        FFI.getLocalOption("floating-window-transparency").let {
+            if (it.isNotEmpty()) {
+                try {
+                    val transparency = it.toInt()
+                    if (transparency in 0..10) {
+                        viewTransparency = transparency * 1f / 10
+                    }
+                } catch (e: Exception) {
+                    e.printStackTrace()
+                }
+            }
+        }
+        // custom svg
+        FFI.getLocalOption("floating-window-svg").let {
+            if (it.isNotEmpty()) {
+                customSvg = it
+            }
+        }
+        // position
+        lastLayoutX = 0
+        lastLayoutY = (wh.second - viewHeight) / 2
+    }
+
+    override fun onDestroy() {
+        super.onDestroy()
+        windowManager.removeView(floatingView)
+    }
+
+    private fun performClick() {
+        showPopupMenu()
+    }
+
+    override fun onTouch(view: View?, event: MotionEvent?): Boolean {
+        if (viewTransparency == 0f) return false
+        when (event?.action) {
+            MotionEvent.ACTION_DOWN -> {
+                dragging = false
+                lastDownX = event.rawX
+                lastDownY = event.rawY
+            }
+            MotionEvent.ACTION_UP -> {
+                val clickDragTolerance = 10f
+                if (abs(event.rawX - lastDownX) < clickDragTolerance && abs(event.rawY - lastDownY) < clickDragTolerance) {
+                    performClick()
+                } else {
+                    moveToScreenSide()
+                }
+            }
+            MotionEvent.ACTION_MOVE -> {
+                val dx = event.rawX - lastDownX
+                val dy = event.rawY - lastDownY
+                // ignore too small fist start moving(some time is click)
+                if (!dragging && dx*dx+dy*dy < 25) {
+                    return false
+                }
+                dragging = true
+                layoutParams.x = event.rawX.toInt()
+                layoutParams.y = event.rawY.toInt()
+                layoutParams.width = viewWidth
+                floatingView.setImageDrawable(originalDrawable)
+                windowManager.updateViewLayout(view, layoutParams)
+                lastLayoutX = layoutParams.x
+                lastLayoutY = layoutParams.y
+            }
+        }
+        return false
+    }
+
+    private fun moveToScreenSide(center: Boolean = false) {
+        val windowManager = getSystemService(WINDOW_SERVICE) as WindowManager
+        val wh = getScreenSize(windowManager)
+        val w = wh.first
+        if (layoutParams.x < w / 2) {
+            layoutParams.x = 0
+            floatingView.setImageDrawable(rightHalfDrawable)
+        } else {
+            layoutParams.x = w - viewWidth / 2
+            floatingView.setImageDrawable(leftHalfDrawable)
+        }
+        if (center) {
+            layoutParams.y = (wh.second - viewHeight) / 2
+        }
+        layoutParams.width = viewWidth / 2
+        windowManager.updateViewLayout(floatingView, layoutParams)
+        lastLayoutX = layoutParams.x
+        lastLayoutY = layoutParams.y
+    }
+
+    override fun onConfigurationChanged(newConfig: Configuration) {
+        super.onConfigurationChanged(newConfig)
+        moveToScreenSide(true)
+    }
+
+     private fun showPopupMenu() {
+         val popupMenu = PopupMenu(this, floatingView)
+         val idShowRustDesk = 0
+         popupMenu.menu.add(0, idShowRustDesk, 0, translate("Show RustDesk"))
+         val idStopService = 1
+         popupMenu.menu.add(0, idStopService, 0, translate("Stop service"))
+         popupMenu.setOnMenuItemClickListener { menuItem ->
+             when (menuItem.itemId) {
+                 idShowRustDesk -> {
+                     openMainActivity()
+                     true
+                 }
+                 idStopService -> {
+                     stopMainService()
+                     true
+                 }
+                 else -> false
+             }
+         }
+         popupMenu.setOnDismissListener {
+             moveToScreenSide()
+         }
+         popupMenu.show()
+     }
+
+
+    private fun openMainActivity() {
+        val intent = Intent(this, MainActivity::class.java)
+        intent.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
+        val pendingIntent = PendingIntent.getActivity(
+            this, 0, intent,
+            PendingIntent.FLAG_IMMUTABLE or PendingIntent.FLAG_ONE_SHOT
+        )
+        try {
+            pendingIntent.send()
+        } catch (e: PendingIntent.CanceledException) {
+            e.printStackTrace()
+        }
+    }
+
+    private fun stopMainService() {
+        MainActivity.flutterMethodChannel?.invokeMethod("stop_service", null)
+    }
+}
+

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt

@@ -252,19 +252,9 @@ class MainActivity : FlutterActivity() {
         val codecArray = JSONArray()
 
         val windowManager = getSystemService(Context.WINDOW_SERVICE) as WindowManager
-        var w = 0
+        val wh = getScreenSize(windowManager)
-        var h = 0
+        var w = wh.first
-        @Suppress("DEPRECATION")
+        var h = wh.second
-        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
-            val m = windowManager.maximumWindowMetrics
-            w = m.bounds.width()
-            h = m.bounds.height()
-        } else {
-            val dm = DisplayMetrics()
-            windowManager.defaultDisplay.getRealMetrics(dm)
-            w = dm.widthPixels
-            h = dm.heightPixels
-        }
         val align = 64
         w = (w + align - 1) / align * align
         h = (h + align - 1) / align * align
@@ -374,4 +364,21 @@ class MainActivity : FlutterActivity() {
             Log.d(logTag, "onVoiceCallClosed success")
         }
     }
+
+    private var disableFloatingWindow: Boolean? = null
+    override fun onStop() {
+        super.onStop()
+        if (disableFloatingWindow == null) {
+            disableFloatingWindow = FFI.getLocalOption("disable-floating-window") == "Y"
+            Log.d(logTag, "disableFloatingWindow: $disableFloatingWindow")
+        }
+        if (disableFloatingWindow != true && MainService.isReady) {
+            startService(Intent(this, FloatingWindowService::class.java))
+        }
+    }
+
+    override fun onStart() {
+        super.onStart()
+        stopService(Intent(this, FloatingWindowService::class.java))
+    }
 }

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainService.kt

@@ -191,11 +191,6 @@ class MainService : Service() {
     private val powerManager: PowerManager by lazy { applicationContext.getSystemService(Context.POWER_SERVICE) as PowerManager }
     private val wakeLock: PowerManager.WakeLock by lazy { powerManager.newWakeLock(PowerManager.ACQUIRE_CAUSES_WAKEUP or PowerManager.SCREEN_BRIGHT_WAKE_LOCK, "rustdesk:wakelock")}
 
-    private fun translate(input: String): String {
-        Log.d(logTag, "translate:$LOCAL_NAME")
-        return FFI.translateLocale(LOCAL_NAME, input)
-    }
-
     companion object {
         private var _isReady = false // media permission ready status
         private var _isStart = false // screen capture start status
@@ -486,6 +481,7 @@ class MainService : Service() {
         mediaProjection = null
         checkMediaPermission()
         stopForeground(true)
+        stopService(Intent(this, FloatingWindowService::class.java))
         stopSelf()
     }
 

flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/common.kt

@@ -15,10 +15,14 @@ import android.os.Looper
 import android.os.PowerManager
 import android.provider.Settings
 import android.provider.Settings.*
+import android.util.DisplayMetrics
+import android.util.Log
+import android.view.WindowManager
 import androidx.annotation.RequiresApi
 import androidx.core.content.ContextCompat.getSystemService
 import com.hjq.permissions.Permission
 import com.hjq.permissions.XXPermissions
+import ffi.FFI
 import java.nio.ByteBuffer
 import java.util.*
 
@@ -120,3 +124,26 @@ class AudioReader(val bufSize: Int, private val maxFrames: Int) {
         }
     }
 }
+
+
+fun getScreenSize(windowManager: WindowManager) : Pair<Int, Int>{
+    var w = 0
+    var h = 0
+    @Suppress("DEPRECATION")
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+        val m = windowManager.maximumWindowMetrics
+        w = m.bounds.width()
+        h = m.bounds.height()
+    } else {
+        val dm = DisplayMetrics()
+        windowManager.defaultDisplay.getRealMetrics(dm)
+        w = dm.widthPixels
+        h = dm.heightPixels
+    }
+    return Pair(w, h)
+}
+
+ fun translate(input: String): String {
+    Log.d("common", "translate:$LOCAL_NAME")
+    return FFI.translateLocale(LOCAL_NAME, input)
+}

flutter/android/app/src/main/kotlin/ffi.kt

@@ -19,4 +19,5 @@ object FFI {
     external fun refreshScreen()
     external fun setFrameRawEnable(name: String, value: Boolean)
     external fun setCodecInfo(info: String)
+    external fun getLocalOption(key: String): String
 }

flutter/android/app/src/main/res/drawable/floating_window.xml

@@ -0,0 +1,7 @@
+<vector xmlns:aapt="http://schemas.android.com/aapt" xmlns:android="http://schemas.android.com/apk/res/android" android:height="320dp" android:viewportHeight="32" android:viewportWidth="32" android:width="320dp">
+      
+    <path android:fillColor="#ffffff" android:pathData="M16,0L16,0A16,16 0,0 1,32 16L32,16A16,16 0,0 1,16 32L16,32A16,16 0,0 1,0 16L0,16A16,16 0,0 1,16 0z" android:strokeColor="#00000000" android:strokeWidth="1"/>
+      
+    <path android:fillColor="#1a1a1a" android:pathData="m23.89,10.135 l-1.807,1.795c-0.318,0.285 -0.472,0.744 -0.293,1.131 1.204,2.518 0.747,5.52 -1.228,7.494 -1.976,1.973 -4.981,2.429 -7.502,1.226 -0.371,-0.166 -0.807,-0.025 -1.093,0.265l-1.836,1.833c-0.216,0.211 -0.322,0.51 -0.288,0.809 0.034,0.3 0.206,0.567 0.463,0.723 4.326,2.618 9.882,1.951 13.463,-1.618 3.581,-3.568 4.264,-9.115 1.655,-13.443 -0.15,-0.263 -0.414,-0.442 -0.714,-0.484 -0.3,-0.043 -0.603,0.058 -0.819,0.269zM8.265,8.184c-3.599,3.554 -4.304,9.103 -1.709,13.441 0.15,0.264 0.413,0.443 0.714,0.485 0.3,0.042 0.603,-0.058 0.82,-0.27l1.797,-1.785c0.325,-0.285 0.484,-0.749 0.303,-1.141 -1.204,-2.518 -0.748,-5.52 1.228,-7.493 1.975,-1.973 4.981,-2.429 7.502,-1.227 0.367,0.165 0.797,0.028 1.084,-0.254l1.846,-1.844c0.216,-0.211 0.322,-0.509 0.288,-0.809 -0.035,-0.299 -0.206,-0.566 -0.463,-0.723 -4.334,-2.596 -9.881,-1.908 -13.448,1.668z" android:strokeWidth="0.987992"/>
+    
+</vector>

flutter/lib/consts.dart

@@ -136,6 +136,8 @@ const String kOptionAllowRemoveWallpaper = "allow-remove-wallpaper";
 
 const String kOptionToggleViewOnly = "view-only";
 
+const String kOptionDisableFloatingWindow = "disable-floating-window";
+
 const String kUrlActionClose = "close";
 
 const String kTabLabelHomePage = "Home";

flutter/lib/mobile/pages/server_page.dart

@@ -854,6 +854,15 @@ void androidChannelInit() {
             msgBox(gFFI.sessionId, type, title, text, link, gFFI.dialogManager);
             break;
           }
+        case "stop_service":
+          {
+            print(
+                "stop_service by kotlin, isStart:${gFFI.serverModel.isStart}");
+            if (gFFI.serverModel.isStart) {
+              gFFI.serverModel.stopService();
+            }
+            break;
+          }
       }
     } catch (e) {
       debugPrintStack(label: "MethodCallHandler err:$e");

flutter/lib/models/server_model.dart

@@ -340,6 +340,20 @@ class ServerModel with ChangeNotifier {
     return res;
   }
 
+  Future<bool> checkFloatingWindowPermission() async {
+    debugPrint("androidVersion $androidVersion");
+    if (androidVersion < 23) {
+      return false;
+    }
+    if (await AndroidPermissionManager.check(kSystemAlertWindow)) {
+      debugPrint("alert window permission already granted");
+      return true;
+    }
+    var res = await AndroidPermissionManager.request(kSystemAlertWindow);
+    debugPrint("alert window permission request result: $res");
+    return res;
+  }
+
   /// Toggle the screen sharing service.
   toggleService() async {
     if (_isStart) {
@@ -367,6 +381,9 @@ class ServerModel with ChangeNotifier {
       }
     } else {
       await checkRequestNotificationPermission();
+      if (bind.mainGetLocalOption(key: kOptionDisableFloatingWindow) != 'Y') {
+        await checkFloatingWindowPermission();
+      }
       if (!await AndroidPermissionManager.check(kManageExternalStorage)) {
         await AndroidPermissionManager.request(kManageExternalStorage);
       }

libs/hbb_common/src/config.rs

@@ -2093,6 +2093,12 @@ pub mod keys {
     pub const OPTION_FLUTTER_PEER_CARD_UI_TYLE: &str = "peer-card-ui-type";
     pub const OPTION_FLUTTER_CURRENT_AB_NAME: &str = "current-ab-name";
 
+    // android floating window options
+    pub const OPTION_DISABLE_FLOATING_WINDOW: &str = "disable-floating-window";
+    pub const OPTION_FLOATING_WINDOW_SIZE: &str = "floating-window-size";
+    pub const OPTION_FLOATING_WINDOW_TRANSPARENCY: &str = "floating-window-transparency";
+    pub const OPTION_FLOATING_WINDOW_SVG: &str = "floating-window-svg";
+
     // proxy settings
     // The following options are not real keys, they are just used for custom client advanced settings.
     // The real keys are in Config2::socks.
@@ -2148,6 +2154,10 @@ pub mod keys {
         OPTION_FLUTTER_PEER_TAB_VISIBLE,
         OPTION_FLUTTER_PEER_CARD_UI_TYLE,
         OPTION_FLUTTER_CURRENT_AB_NAME,
+        OPTION_DISABLE_FLOATING_WINDOW,
+        OPTION_FLOATING_WINDOW_SIZE,
+        OPTION_FLOATING_WINDOW_TRANSPARENCY,
+        OPTION_FLOATING_WINDOW_SVG,
     ];
     // DEFAULT_SETTINGS, OVERWRITE_SETTINGS
     pub const KEYS_SETTINGS: &[&str] = &[

src/flutter_ffi.rs

@@ -2224,24 +2224,35 @@ pub mod server_side {
         input: JString,
     ) -> jstring {
         let mut env = env;
-        let res =
+        let res = if let (Ok(input), Ok(locale)) = (env.get_string(&input), env.get_string(&locale))
-            env.with_local_frame_returning_local(10, |env: &mut JNIEnv| -> JniResult<JObject> {
+        {
-                let res = if let (Ok(input), Ok(locale)) =
+            let input: String = input.into();
-                    (env.get_string(&input), env.get_string(&locale))
+            let locale: String = locale.into();
-                {
+            crate::client::translate_locale(input, &locale)
-                    let input: String = input.into();
+        } else {
-                    let locale: String = locale.into();
+            "".into()
-                    crate::client::translate_locale(input, &locale)
+        };
-                } else {
+        return env.new_string(res).unwrap_or(input).into_raw();
-                    "".into()
-                };
-                env.new_string(res).map(|v| v.into())
-            });
-        res.unwrap_or(input.into()).into_raw()
     }
 
     #[no_mangle]
     pub unsafe extern "system" fn Java_ffi_FFI_refreshScreen(_env: JNIEnv, _class: JClass) {
         crate::server::video_service::refresh()
     }
+
+    #[no_mangle]
+    pub unsafe extern "system" fn Java_ffi_FFI_getLocalOption(
+        env: JNIEnv,
+        _class: JClass,
+        key: JString,
+    ) -> jstring {
+        let mut env = env;
+        let res = if let Ok(key) = env.get_string(&key) {
+            let key: String = key.into();
+            super::get_local_option(key)
+        } else {
+            "".into()
+        };
+        return env.new_string(res).unwrap_or_default().into_raw();
+    }
 }