Merge pull request #2317 from fufesou/fix_wayland_ubuntu_22

Fix wayland ubuntu 22

libs/enigo/src/lib.rs

@@ -118,6 +118,13 @@ pub enum MouseButton {
 /// Representing an interface and a set of mouse functions every
 /// operating system implementation _should_ implement.
 pub trait MouseControllable {
+    // https://stackoverflow.com/a/33687996
+    /// Offer the ability to confer concrete type.
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    /// Offer the ability to confer concrete type.
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any;
+
     /// Lets the mouse cursor move to the specified x and y coordinates.
     ///
     /// The topleft corner of your monitor screen is x=0 y=0. Move
@@ -425,6 +432,13 @@ pub enum Key {
 /// Representing an interface and a set of keyboard functions every
 /// operating system implementation _should_ implement.
 pub trait KeyboardControllable {
+    // https://stackoverflow.com/a/33687996
+    /// Offer the ability to confer concrete type.
+    fn as_any(&self) -> &dyn std::any::Any;
+
+    /// Offer the ability to confer concrete type.
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any;
+
     /// Types the string parsed with DSL.
     ///
     /// Typing {+SHIFT}hello{-SHIFT} becomes HELLO.

libs/enigo/src/linux/nix_impl.rs

@@ -3,14 +3,17 @@ use crate::{Key, KeyboardControllable, MouseButton, MouseControllable};
 use std::io::Read;
 use tfc::{traits::*, Context as TFC_Context, Key as TFC_Key};
 
+pub type CustomKeyboard = Box<dyn KeyboardControllable + Send>;
+pub type CustomMouce = Box<dyn MouseControllable + Send>;
+
 /// The main struct for handling the event emitting
 // #[derive(Default)]
 pub struct Enigo {
     xdo: EnigoXdo,
     is_x11: bool,
     tfc: Option<TFC_Context>,
-    uinput_keyboard: Option<Box<dyn KeyboardControllable + Send>>,
+    custom_keyboard: Option<CustomKeyboard>,
-    uinput_mouse: Option<Box<dyn MouseControllable + Send>>,
+    cutsom_mouse: Option<CustomMouce>,
 }
 
 impl Enigo {
@@ -22,16 +25,21 @@ impl Enigo {
     pub fn set_delay(&mut self, delay: u64) {
         self.xdo.set_delay(delay)
     }
-    /// Set uinput keyboard.
+    /// Set custom keyboard.
-    pub fn set_uinput_keyboard(
+    pub fn set_custom_keyboard(&mut self, custom_keyboard: CustomKeyboard) {
-        &mut self,
+        self.custom_keyboard = Some(custom_keyboard)
-        uinput_keyboard: Option<Box<dyn KeyboardControllable + Send>>,
-    ) {
-        self.uinput_keyboard = uinput_keyboard
     }
-    /// Set uinput mouse.
+    /// Set custom mouse.
-    pub fn set_uinput_mouse(&mut self, uinput_mouse: Option<Box<dyn MouseControllable + Send>>) {
+    pub fn set_custom_mouse(&mut self, custom_mouse: CustomMouce) {
-        self.uinput_mouse = uinput_mouse
+        self.cutsom_mouse = Some(custom_mouse)
+    }
+    /// Get custom keyboard.
+    pub fn get_custom_keyboard(&mut self) -> &mut Option<CustomKeyboard> {
+        &mut self.custom_keyboard
+    }
+    /// Get custom mouse.
+    pub fn get_custom_mouse(&mut self) -> &mut Option<CustomMouce> {
+        &mut self.cutsom_mouse
     }
 
     fn tfc_key_down_or_up(&mut self, key: Key, down: bool, up: bool) -> bool {
@@ -84,19 +92,27 @@ impl Default for Enigo {
             } else {
                 None
             },
-            uinput_keyboard: None,
+            custom_keyboard: None,
-            uinput_mouse: None,
+            cutsom_mouse: None,
             xdo: EnigoXdo::default(),
         }
     }
 }
 
 impl MouseControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn mouse_move_to(&mut self, x: i32, y: i32) {
         if self.is_x11 {
             self.xdo.mouse_move_to(x, y);
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_move_to(x, y)
             }
         }
@@ -105,7 +121,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_move_relative(x, y);
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_move_relative(x, y)
             }
         }
@@ -114,7 +130,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_down(button)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_down(button)
             } else {
                 Ok(())
@@ -125,7 +141,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_up(button)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_up(button)
             }
         }
@@ -134,7 +150,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_click(button)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_click(button)
             }
         }
@@ -143,7 +159,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_scroll_x(length)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_scroll_x(length)
             }
         }
@@ -152,7 +168,7 @@ impl MouseControllable for Enigo {
         if self.is_x11 {
             self.xdo.mouse_scroll_y(length)
         } else {
-            if let Some(mouse) = &mut self.uinput_mouse {
+            if let Some(mouse) = &mut self.cutsom_mouse {
                 mouse.mouse_scroll_y(length)
             }
         }
@@ -180,11 +196,19 @@ fn get_led_state(key: Key) -> bool {
 }
 
 impl KeyboardControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn get_key_state(&mut self, key: Key) -> bool {
         if self.is_x11 {
             self.xdo.get_key_state(key)
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.get_key_state(key)
             } else {
                 get_led_state(key)
@@ -196,7 +220,7 @@ impl KeyboardControllable for Enigo {
         if self.is_x11 {
             self.xdo.key_sequence(sequence)
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.key_sequence(sequence)
             }
         }
@@ -211,7 +235,7 @@ impl KeyboardControllable for Enigo {
                 Ok(())
             }
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.key_down(key)
             } else {
                 Ok(())
@@ -225,7 +249,7 @@ impl KeyboardControllable for Enigo {
                 self.xdo.key_up(key)
             }
         } else {
-            if let Some(keyboard) = &mut self.uinput_keyboard {
+            if let Some(keyboard) = &mut self.custom_keyboard {
                 keyboard.key_up(key)
             }
         }

libs/enigo/src/linux/xdo.rs

@@ -102,6 +102,14 @@ impl Drop for EnigoXdo {
     }
 }
 impl MouseControllable for EnigoXdo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn mouse_move_to(&mut self, x: i32, y: i32) {
         if self.xdo.is_null() {
             return;
@@ -277,6 +285,14 @@ fn keysequence<'a>(key: Key) -> Cow<'a, str> {
     })
 }
 impl KeyboardControllable for EnigoXdo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn get_key_state(&mut self, key: Key) -> bool {
         if self.xdo.is_null() {
             return false;

libs/enigo/src/macos/macos_impl.rs

@@ -163,6 +163,14 @@ impl Default for Enigo {
 }
 
 impl MouseControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn mouse_move_to(&mut self, x: i32, y: i32) {
         let pressed = Self::pressed_buttons();
 
@@ -319,6 +327,14 @@ impl MouseControllable for Enigo {
 // com/questions/1918841/how-to-convert-ascii-character-to-cgkeycode
 
 impl KeyboardControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+    
     fn key_sequence(&mut self, sequence: &str) {
         // NOTE(dustin): This is a fix for issue https://github.com/enigo-rs/enigo/issues/68
         // TODO(dustin): This could be improved by aggregating 20 bytes worth of graphemes at a time

libs/enigo/src/win/win_impl.rs

@@ -105,6 +105,14 @@ fn get_error() -> String {
 }
 
 impl MouseControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+
     fn mouse_move_to(&mut self, x: i32, y: i32) {
         mouse_event(
             MOUSEEVENTF_MOVE | MOUSEEVENTF_ABSOLUTE | MOUSEEVENTF_VIRTUALDESK,
@@ -170,6 +178,14 @@ impl MouseControllable for Enigo {
 }
 
 impl KeyboardControllable for Enigo {
+    fn as_any(&self) -> &dyn std::any::Any {
+        self
+    }
+
+    fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+        self
+    }
+    
     fn key_sequence(&mut self, sequence: &str) {
         let mut buffer = [0; 2];
 

libs/scrap/src/wayland/pipewire.rs

@@ -16,9 +16,10 @@ use gstreamer as gst;
 use gstreamer::prelude::*;
 use gstreamer_app::AppSink;
 
+use hbb_common::config;
+
 use super::capturable::PixelProvider;
 use super::capturable::{Capturable, Recorder};
-
 use super::pipewire_dbus::{OrgFreedesktopPortalRequestResponse, OrgFreedesktopPortalScreenCast};
 
 #[derive(Debug, Clone, Copy)]
@@ -121,6 +122,7 @@ pub struct PipeWireRecorder {
     appsink: AppSink,
     width: usize,
     height: usize,
+    saved_raw_data: Vec<u128>, // for faster compare and copy
 }
 
 impl PipeWireRecorder {
@@ -130,6 +132,7 @@ impl PipeWireRecorder {
         let src = gst::ElementFactory::make("pipewiresrc", None)?;
         src.set_property("fd", &capturable.fd.as_raw_fd())?;
         src.set_property("path", &format!("{}", capturable.path))?;
+        src.set_property("keepalive_time", &1_000.as_raw_fd())?;
 
         // For some reason pipewire blocks on destruction of AppSink if this is not set to true,
         // see: https://gitlab.freedesktop.org/pipewire/pipewire/-/issues/982
@@ -158,6 +161,7 @@ impl PipeWireRecorder {
             height: 0,
             buffer_cropped: vec![],
             is_cropped: false,
+            saved_raw_data: Vec::new(),
         })
     }
 }
@@ -190,6 +194,9 @@ impl Recorder for PipeWireRecorder {
             let buf = buf
                 .into_mapped_buffer_readable()
                 .map_err(|_| GStreamerError("Failed to map buffer.".into()))?;
+            if let Err(..) = crate::would_block_if_equal(&mut self.saved_raw_data, buf.as_slice()) {
+                return Ok(PixelProvider::NONE);
+            }
             let buf_size = buf.get_size();
             // BGRx is 4 bytes per pixel
             if buf_size != (w * h * 4) {
@@ -384,6 +391,8 @@ fn streams_from_response(response: OrgFreedesktopPortalRequestResponse) -> Vec<P
 }
 
 static mut INIT: bool = false;
+const RESTORE_TOKEN: &str = "restore_token";
+const RESTORE_TOKEN_CONF_KEY: &str = "wayland-restore-token";
 
 // mostly inspired by https://gitlab.gnome.org/snippets/19
 fn request_screen_cast(
@@ -412,6 +421,12 @@ fn request_screen_cast(
         "handle_token".to_string(),
         Variant(Box::new("u1".to_string())),
     );
+    // The following code may be improved.
+    // https://flatpak.github.io/xdg-desktop-portal/#:~:text=To%20avoid%20a%20race%20condition
+    // To avoid a race condition
+    // between the caller subscribing to the signal after receiving the reply for the method call and the signal getting emitted,
+    // a convention for Request object paths has been established that allows
+    // the caller to subscribe to the signal before making the method call.
     let path = portal.create_session(args)?;
     handle_response(
         &conn,
@@ -419,6 +434,16 @@ fn request_screen_cast(
         move |r: OrgFreedesktopPortalRequestResponse, c, _| {
             let portal = get_portal(c);
             let mut args: PropMap = HashMap::new();
+            if let Ok(version) = portal.version() {
+                if version >= 4 {
+                    let restore_token = config::LocalConfig::get_option(RESTORE_TOKEN_CONF_KEY);
+                    if !restore_token.is_empty() {
+                        args.insert(RESTORE_TOKEN.to_string(), Variant(Box::new(restore_token)));
+                    }
+                    // persist_mode may be configured by the user.
+                    args.insert("persist_mode".to_string(), Variant(Box::new(2u32)));
+                }
+            }
             args.insert(
                 "handle_token".to_string(),
                 Variant(Box::new("u2".to_string())),
@@ -476,12 +501,24 @@ fn request_screen_cast(
                         c,
                         path,
                         move |r: OrgFreedesktopPortalRequestResponse, c, _| {
+                            let portal = get_portal(c);
+                            if let Ok(version) = portal.version() {
+                                if version >= 4 {
+                                    if let Some(restore_token) = r.results.get(RESTORE_TOKEN) {
+                                        if let Some(restore_token) = restore_token.as_str() {
+                                            config::LocalConfig::set_option(
+                                                RESTORE_TOKEN_CONF_KEY.to_owned(),
+                                                restore_token.to_owned(),
+                                            );
+                                        }
+                                    }
+                                }
+                            }
                             streams
                                 .clone()
                                 .lock()
                                 .unwrap()
                                 .append(&mut streams_from_response(r));
-                            let portal = get_portal(c);
                             fd.clone().lock().unwrap().replace(
                                 portal.open_pipe_wire_remote(session.clone(), HashMap::new())?,
                             );

src/ipc.rs

@@ -117,6 +117,7 @@ pub enum DataMouse {
     Click(enigo::MouseButton),
     ScrollX(i32),
     ScrollY(i32),
+    Refresh,
 }
 
 #[derive(Debug, Serialize, Deserialize, Clone)]

src/server.rs

@@ -1,6 +1,8 @@
 use crate::ipc::Data;
 use bytes::Bytes;
 pub use connection::*;
+#[cfg(not(any(target_os = "android", target_os = "ios")))]
+use hbb_common::config::Config2;
 use hbb_common::{
     allow_err,
     anyhow::{anyhow, Context},
@@ -15,8 +17,6 @@ use hbb_common::{
     timeout, tokio, ResultType, Stream,
 };
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
-use hbb_common::config::Config2;
-#[cfg(not(any(target_os = "android", target_os = "ios")))]
 use service::ServiceTmpl;
 use service::{GenericService, Service, Subscriber};
 use std::{
@@ -377,6 +377,7 @@ pub async fn start_server(is_server: bool) {
         #[cfg(windows)]
         crate::platform::windows::bootstrap();
         input_service::fix_key_down_timeout_loop();
+        allow_err!(video_service::check_init().await);
         #[cfg(target_os = "macos")]
         tokio::spawn(async { sync_and_watch_config_dir().await });
         crate::RendezvousMediator::start_all().await;

src/server/input_service.rs

@@ -219,19 +219,43 @@ lazy_static::lazy_static! {
     static ref IS_SERVER: bool =  std::env::args().nth(1) == Some("--server".to_owned());
 }
 
+// First call set_uinput() will create keyboard and mouse clients.
+// The clients are ipc connections that must live shorter than tokio runtime.
+// Thus this funtion must not be called in a temporary runtime.
 #[cfg(target_os = "linux")]
 pub async fn set_uinput() -> ResultType<()> {
     // Keyboard and mouse both open /dev/uinput
     // TODO: Make sure there's no race
-    let keyboard = super::uinput::client::UInputKeyboard::new().await?;
-    log::info!("UInput keyboard created");
-    let mouse = super::uinput::client::UInputMouse::new().await?;
-    log::info!("UInput mouse created");
 
-    let xxx = ENIGO.lock();
+    if ENIGO.lock().unwrap().get_custom_keyboard().is_none() {
-    let mut en = xxx.unwrap();
+        let keyboard = super::uinput::client::UInputKeyboard::new().await?;
-    en.set_uinput_keyboard(Some(Box::new(keyboard)));
+        log::info!("UInput keyboard created");
-    en.set_uinput_mouse(Some(Box::new(mouse)));
+        ENIGO
+            .lock()
+            .unwrap()
+            .set_custom_keyboard(Box::new(keyboard));
+    }
+
+    let mouse_created = ENIGO.lock().unwrap().get_custom_mouse().is_some();
+    if mouse_created {
+        std::thread::spawn(|| {
+            if let Some(mouse) = ENIGO.lock().unwrap().get_custom_mouse() {
+                if let Some(mouse) = mouse
+                    .as_mut_any()
+                    .downcast_mut::<super::uinput::client::UInputMouse>()
+                {
+                    allow_err!(mouse.send_refresh());
+                } else {
+                    log::error!("failed downcast uinput mouse");
+                }
+            }
+        });
+    } else {
+        let mouse = super::uinput::client::UInputMouse::new().await?;
+        log::info!("UInput mouse created");
+        ENIGO.lock().unwrap().set_custom_mouse(Box::new(mouse));
+    }
+
     Ok(())
 }
 

src/server/uinput.rs

@@ -63,6 +63,14 @@ pub mod client {
     }
 
     impl KeyboardControllable for UInputKeyboard {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+            self
+        }
+
         fn get_key_state(&mut self, key: Key) -> bool {
             match self.send_get_key_state(Data::Keyboard(DataKeyboard::GetKeyState(key))) {
                 Ok(state) => state,
@@ -105,9 +113,21 @@ pub mod client {
         async fn send(&mut self, data: Data) -> ResultType<()> {
             self.conn.send(&data).await
         }
+
+        pub fn send_refresh(&mut self) -> ResultType<()> {
+            self.send(Data::Mouse(DataMouse::Refresh))
+        }
     }
 
     impl MouseControllable for UInputMouse {
+        fn as_any(&self) -> &dyn std::any::Any {
+            self
+        }
+
+        fn as_mut_any(&mut self) -> &mut dyn std::any::Any {
+            self
+        }
+
         fn mouse_move_to(&mut self, x: i32, y: i32) {
             allow_err!(self.send(Data::Mouse(DataMouse::MoveTo(x, y))));
         }
@@ -492,6 +512,9 @@ pub mod service {
                     allow_err!(mouse.scroll_wheel(&scroll))
                 }
             }
+            DataMouse::Refresh => {
+                // unreachable!()
+            }
         }
     }
 
@@ -562,7 +585,17 @@ pub mod service {
                             Ok(Some(data)) => {
                                 match data {
                                     Data::Mouse(data) => {
-                                        handle_mouse(&mut mouse, &data);
+                                        if let DataMouse::Refresh = data {
+                                            mouse = match mouce::Mouse::new_uinput(rng_x, rng_y) {
+                                                Ok(mouse) => mouse,
+                                                Err(e) => {
+                                                    log::error!("Failed to create mouse, {}", e);
+                                                    return;
+                                                }
+                                            }
+                                        } else {
+                                            handle_mouse(&mut mouse, &data);
+                                        }
                                     }
                                     _ => {
                                     }

src/server/video_service.rs

@@ -599,14 +599,11 @@ fn run(sp: GenericService) -> ResultType<()> {
                     }
                     try_gdi += 1;
                 }
-
                 #[cfg(target_os = "linux")]
                 {
                     would_block_count += 1;
                     if !scrap::is_x11() {
                         if would_block_count >= 100 {
-                            // For now, the user should choose and agree screen sharing agiain.
-                            // to-do: Remember choice, attendless...
                             super::wayland::release_resouce();
                             bail!("Wayland capturer none 100 times, try restart captuere");
                         }
@@ -762,6 +759,16 @@ fn get_display_num() -> usize {
     }
 }
 
+pub async fn check_init() -> ResultType<()> {
+    #[cfg(target_os = "linux")]
+    {
+        if !scrap::is_x11() {
+            return super::wayland::check_init().await;
+        }
+    }
+    Ok(())
+}
+
 pub(super) fn get_displays_2(all: &Vec<Display>) -> (usize, Vec<DisplayInfo>) {
     let mut displays = Vec::new();
     let mut primary = 0;

src/server/wayland.rs

@@ -115,7 +115,7 @@ pub(super) fn is_inited() -> Option<Message> {
     }
 }
 
-async fn check_init() -> ResultType<()> {
+pub(super) async fn check_init() -> ResultType<()> {
     if !scrap::is_x11() {
         let mut minx = 0;
         let mut maxx = 0;
@@ -241,6 +241,7 @@ pub(super) fn get_display_num() -> ResultType<usize> {
     }
 }
 
+#[allow(dead_code)]
 pub(super) fn release_resouce() {
     if scrap::is_x11() {
         return;