fix: clipboard, windows, controlled side, formats (#8885)

* fix: clipboard, windows, controlled side, formats

Signed-off-by: fufesou <linlong1266@gmail.com>

* Clipboard, reuse ipc conn and send_raw()

Signed-off-by: fufesou <linlong1266@gmail.com>

* Clipboard, merge content buffer

Signed-off-by: fufesou <linlong1266@gmail.com>

* refact: clipboard service, ipc stream

Signed-off-by: fufesou <linlong1266@gmail.com>

---------

Signed-off-by: fufesou <linlong1266@gmail.com>

src/clipboard.rs

@@ -16,10 +16,11 @@ lazy_static::lazy_static! {
     static ref ARBOARD_MTX: Arc<Mutex<()>> = Arc::new(Mutex::new(()));
     // cache the clipboard msg
     static ref LAST_MULTI_CLIPBOARDS: Arc<Mutex<MultiClipboards>> = Arc::new(Mutex::new(MultiClipboards::new()));
-    // Clipboard on Linux is "server--clients" mode. 
+    // For updating in server and getting content in cm.
+    // Clipboard on Linux is "server--clients" mode.
     // The clipboard content is owned by the server and passed to the clients when requested.
     // Plain text is the only exception, it does not require the server to be present.
-    static ref CLIPBOARD_UPDATE_CTX: Arc<Mutex<Option<ClipboardContext>>> = Arc::new(Mutex::new(None));
+    static ref CLIPBOARD_CTX: Arc<Mutex<Option<ClipboardContext>>> = Arc::new(Mutex::new(None));
 }
 
 const SUPPORTED_FORMATS: &[ClipboardFormat] = &[
@@ -159,12 +160,34 @@ pub fn check_clipboard(
     None
 }
 
+#[cfg(target_os = "windows")]
+pub fn check_clipboard_cm() -> ResultType<MultiClipboards> {
+    let mut ctx = CLIPBOARD_CTX.lock().unwrap();
+    if ctx.is_none() {
+        match ClipboardContext::new() {
+            Ok(x) => {
+                *ctx = Some(x);
+            }
+            Err(e) => {
+                hbb_common::bail!("Failed to create clipboard context: {}", e);
+            }
+        }
+    }
+    if let Some(ctx) = ctx.as_mut() {
+        let content = ctx.get(ClipboardSide::Host, false)?;
+        let clipboards = proto::create_multi_clipboards(content);
+        Ok(clipboards)
+    } else {
+        hbb_common::bail!("Failed to create clipboard context");
+    }
+}
+
 fn update_clipboard_(multi_clipboards: Vec<Clipboard>, side: ClipboardSide) {
     let mut to_update_data = proto::from_multi_clipbards(multi_clipboards);
     if to_update_data.is_empty() {
         return;
     }
-    let mut ctx = CLIPBOARD_UPDATE_CTX.lock().unwrap();
+    let mut ctx = CLIPBOARD_CTX.lock().unwrap();
     if ctx.is_none() {
         match ClipboardContext::new() {
             Ok(x) => {

src/ipc.rs

@@ -1,10 +1,3 @@
-use std::{
-    collections::HashMap,
-    sync::atomic::{AtomicBool, Ordering},
-};
-#[cfg(not(windows))]
-use std::{fs::File, io::prelude::*};
-
 use crate::{
     privacy_mode::PrivacyModeState,
     ui_interface::{get_local_option, set_local_option},
@@ -14,6 +7,12 @@ use parity_tokio_ipc::{
     Connection as Conn, ConnectionClient as ConnClient, Endpoint, Incoming, SecurityAttributes,
 };
 use serde_derive::{Deserialize, Serialize};
+use std::{
+    collections::HashMap,
+    sync::atomic::{AtomicBool, Ordering},
+};
+#[cfg(not(windows))]
+use std::{fs::File, io::prelude::*};
 
 #[cfg(all(feature = "flutter", feature = "plugin_framework"))]
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
@@ -26,8 +25,11 @@ use hbb_common::{
     config::{self, Config, Config2},
     futures::StreamExt as _,
     futures_util::sink::SinkExt,
-    log, password_security as password, timeout, tokio,
+    log, password_security as password, timeout,
-    tokio::io::{AsyncRead, AsyncWrite},
+    tokio::{
+        self,
+        io::{AsyncRead, AsyncWrite},
+    },
     tokio_util::codec::Framed,
     ResultType,
 };
@@ -100,6 +102,20 @@ pub enum FS {
     },
 }
 
+#[cfg(target_os = "windows")]
+#[derive(Debug, Serialize, Deserialize, Clone)]
+#[serde(tag = "t")]
+pub struct ClipboardNonFile {
+    pub compress: bool,
+    pub content: bytes::Bytes,
+    pub content_len: usize,
+    pub next_raw: bool,
+    pub width: i32,
+    pub height: i32,
+    // message.proto: ClipboardFormat
+    pub format: i32,
+}
+
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 #[derive(Debug, Serialize, Deserialize, Clone)]
 #[serde(tag = "t", content = "c")]
@@ -207,6 +223,8 @@ pub enum Data {
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     ClipboardFile(ClipboardFile),
     ClipboardFileEnabled(bool),
+    #[cfg(target_os = "windows")]
+    ClipboardNonFile(Option<(String, Vec<ClipboardNonFile>)>),
     PrivacyModeState((i32, PrivacyModeState, String)),
     TestRendezvousServer,
     #[cfg(not(any(target_os = "android", target_os = "ios")))]

src/server/clipboard_service.rs

@@ -3,6 +3,8 @@ pub use crate::clipboard::{
     check_clipboard, ClipboardContext, ClipboardSide, CLIPBOARD_INTERVAL as INTERVAL,
     CLIPBOARD_NAME as NAME,
 };
+#[cfg(windows)]
+use crate::ipc::{self, ClipboardFile, ClipboardNonFile, Data};
 use clipboard_master::{CallbackResult, ClipboardHandler};
 use std::{
     io,
@@ -14,6 +16,8 @@ struct Handler {
     sp: EmptyExtraFieldService,
     ctx: Option<ClipboardContext>,
     tx_cb_result: Sender<CallbackResult>,
+    #[cfg(target_os = "windows")]
+    stream: Option<ipc::ConnectionTmpl<parity_tokio_ipc::ConnectionClient>>,
 }
 
 pub fn new() -> GenericService {
@@ -28,6 +32,8 @@ fn run(sp: EmptyExtraFieldService) -> ResultType<()> {
         sp: sp.clone(),
         ctx: Some(ClipboardContext::new()?),
         tx_cb_result,
+        #[cfg(target_os = "windows")]
+        stream: None,
     };
 
     let (tx_start_res, rx_start_res) = channel();
@@ -64,8 +70,10 @@ fn run(sp: EmptyExtraFieldService) -> ResultType<()> {
 impl ClipboardHandler for Handler {
     fn on_clipboard_change(&mut self) -> CallbackResult {
         self.sp.snapshot(|_sps| Ok(())).ok();
-        if let Some(msg) = check_clipboard(&mut self.ctx, ClipboardSide::Host, false) {
+        if self.sp.ok() {
-            self.sp.send(msg);
+            if let Some(msg) = self.get_clipboard_msg() {
+                self.sp.send(msg);
+            }
         }
         CallbackResult::Next
     }
@@ -77,3 +85,107 @@ impl ClipboardHandler for Handler {
         CallbackResult::Next
     }
 }
+
+impl Handler {
+    fn get_clipboard_msg(&mut self) -> Option<Message> {
+        #[cfg(target_os = "windows")]
+        if crate::common::is_server() && crate::platform::is_root() {
+            match self.read_clipboard_from_cm_ipc() {
+                Err(e) => {
+                    log::error!("Failed to read clipboard from cm: {}", e);
+                }
+                Ok(data) => {
+                    let mut msg = Message::new();
+                    let multi_clipboards = MultiClipboards {
+                        clipboards: data
+                            .into_iter()
+                            .map(|c| Clipboard {
+                                compress: c.compress,
+                                content: c.content,
+                                width: c.width,
+                                height: c.height,
+                                format: ClipboardFormat::from_i32(c.format)
+                                    .unwrap_or(ClipboardFormat::Text)
+                                    .into(),
+                                ..Default::default()
+                            })
+                            .collect(),
+                        ..Default::default()
+                    };
+                    msg.set_multi_clipboards(multi_clipboards);
+                    return Some(msg);
+                }
+            }
+        }
+        check_clipboard(&mut self.ctx, ClipboardSide::Host, false)
+    }
+
+    // It's ok to do async operation in the clipboard service because:
+    // 1. the clipboard is not used frequently.
+    // 2. the clipboard handle is sync and will not block the main thread.
+    #[cfg(windows)]
+    #[tokio::main(flavor = "current_thread")]
+    async fn read_clipboard_from_cm_ipc(&mut self) -> ResultType<Vec<ClipboardNonFile>> {
+        let mut is_sent = false;
+        if let Some(stream) = &mut self.stream {
+            // If previous stream is still alive, reuse it.
+            // If the previous stream is dead, `is_sent` will trigger reconnect.
+            is_sent = stream.send(&Data::ClipboardNonFile(None)).await.is_ok();
+        }
+        if !is_sent {
+            let mut stream = crate::ipc::connect(100, "_cm").await?;
+            stream.send(&Data::ClipboardNonFile(None)).await?;
+            self.stream = Some(stream);
+        }
+
+        if let Some(stream) = &mut self.stream {
+            loop {
+                match stream.next_timeout(800).await? {
+                    Some(Data::ClipboardNonFile(Some((err, mut contents)))) => {
+                        if !err.is_empty() {
+                            bail!("{}", err);
+                        } else {
+                            if contents.iter().any(|c| c.next_raw) {
+                                match timeout(1000, stream.next_raw()).await {
+                                    Ok(Ok(mut data)) => {
+                                        for c in &mut contents {
+                                            if c.next_raw {
+                                                if c.content_len <= data.len() {
+                                                    c.content =
+                                                        data.split_off(c.content_len).into();
+                                                } else {
+                                                    // Reconnect the next time to avoid the next raw data mismatch.
+                                                    self.stream = None;
+                                                    bail!("failed to get raw clipboard data: invalid size");
+                                                }
+                                            }
+                                        }
+                                    }
+                                    Ok(Err(e)) => {
+                                        // reset by peer
+                                        self.stream = None;
+                                        bail!("failed to get raw clipboard data: {}", e);
+                                    }
+                                    Err(e) => {
+                                        // Reconnect to avoid the next raw data remaining in the buffer.
+                                        self.stream = None;
+                                        log::debug!("failed to get raw clipboard data: {}", e);
+                                    }
+                                }
+                            }
+                            return Ok(contents);
+                        }
+                    }
+                    Some(Data::ClipboardFile(ClipboardFile::MonitorReady)) => {
+                        // ClipboardFile::MonitorReady is the first message sent by cm.
+                    }
+                    _ => {
+                        bail!("failed to get clipboard data from cm");
+                    }
+                }
+            }
+        }
+        // unreachable!
+        bail!("failed to get clipboard data from cm");
+    }
+}

src/ui_cm_interface.rs

@@ -1,16 +1,5 @@
-#[cfg(any(target_os = "android", target_os = "ios", feature = "flutter"))]
+#[cfg(target_os = "windows")]
-use std::iter::FromIterator;
+use crate::ipc::ClipboardNonFile;
-#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
-use std::sync::Arc;
-use std::{
-    collections::HashMap,
-    ops::{Deref, DerefMut},
-    sync::{
-        atomic::{AtomicI64, Ordering},
-        RwLock,
-    },
-};
-
 #[cfg(not(any(target_os = "android", target_os = "ios")))]
 use crate::ipc::Connection;
 #[cfg(not(any(target_os = "ios")))]
@@ -36,6 +25,18 @@ use hbb_common::{
 #[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
 use hbb_common::{tokio::sync::Mutex as TokioMutex, ResultType};
 use serde_derive::Serialize;
+#[cfg(any(target_os = "android", target_os = "ios", feature = "flutter"))]
+use std::iter::FromIterator;
+#[cfg(any(target_os = "windows", target_os = "linux", target_os = "macos"))]
+use std::sync::Arc;
+use std::{
+    collections::HashMap,
+    ops::{Deref, DerefMut},
+    sync::{
+        atomic::{AtomicI64, Ordering},
+        RwLock,
+    },
+};
 
 #[derive(Serialize, Clone)]
 pub struct Client {
@@ -486,6 +487,41 @@ impl<T: InvokeUiCM> IpcTaskRunner<T> {
                                 Data::CloseVoiceCall(reason) => {
                                     self.cm.voice_call_closed(self.conn_id, reason.as_str());
                                 }
+                                #[cfg(target_os = "windows")]
+                                Data::ClipboardNonFile(_) => {
+                                    match crate::clipboard::check_clipboard_cm() {
+                                        Ok(multi_clipoards) => {
+                                            let mut raw_contents = bytes::BytesMut::new();
+                                            let mut main_data = vec![];
+                                            for c in multi_clipoards.clipboards.into_iter() {
+                                                let (content, content_len, next_raw) = {
+                                                    // TODO: find out a better threshold
+                                                    let content_len = c.content.len();
+                                                    if content_len > 1024 * 3 {
+                                                        (c.content, content_len, false)
+                                                    } else {
+                                                        raw_contents.extend(c.content);
+                                                        (bytes::Bytes::new(), content_len, true)
+                                                    }
+                                                };
+                                                main_data.push(ClipboardNonFile {
+                                                    compress: c.compress,
+                                                    content,
+                                                    content_len,
+                                                    next_raw,
+                                                    width: c.width,
+                                                    height: c.height,
+                                                    format: c.format.value(),
+                                                });
+                                            }
+                                            allow_err!(self.stream.send(&Data::ClipboardNonFile(Some(("".to_owned(), main_data)))).await);
+                                            allow_err!(self.stream.send_raw(raw_contents.into()).await);
+                                        }
+                                        Err(e) => {
+                                            allow_err!(self.stream.send(&Data::ClipboardNonFile(Some((format!("{}", e), vec![])))).await);
+                                        }
+                                    }
+                                }
                                 _ => {
 
                                 }