texture paint

Signed-off-by: fufesou <shuanglongchen@yeah.net>

flutter/lib/desktop/pages/remote_page.dart

@@ -126,9 +126,9 @@ class _RemotePageState extends State<RemotePage>
     // Register texture.
     _textureId.value = -1;
     textureRenderer.createTexture(_textureKey).then((id) async {
+      debugPrint("id: $id, texture_key: $_textureKey");
       if (id != -1) {
         final ptr = await textureRenderer.getTexturePtr(_textureKey);
-        debugPrint("id: $id, texture_key: $_textureKey");
         platformFFI.registerTexture(widget.id, ptr);
         _textureId.value = id;
       }
@@ -197,6 +197,8 @@ class _RemotePageState extends State<RemotePage>
   @override
   void dispose() {
     debugPrint("REMOTE PAGE dispose ${widget.id}");
+    platformFFI.registerTexture(widget.id, 0);
+    textureRenderer.closeTexture(_textureKey);
     // ensure we leave this session, this is a double check
     bind.sessionEnterOrLeave(id: widget.id, enter: false);
     DesktopMultiWindow.removeListener(this);
@@ -212,7 +214,6 @@ class _RemotePageState extends State<RemotePage>
       Wakelock.disable();
     }
     Get.delete<FFI>(tag: widget.id);
-    textureRenderer.closeTexture(_textureKey);
     super.dispose();
     _removeStates(widget.id);
   }
@@ -484,13 +485,12 @@ class _ImagePaintState extends State<ImagePaint> {
       final imageWidth = c.getDisplayWidth() * s;
       final imageHeight = c.getDisplayHeight() * s;
       final imageSize = Size(imageWidth, imageHeight);
-      print("width: $imageWidth/$imageHeight");
       // final imageWidget = CustomPaint(
       //   size: imageSize,
       //   painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s),
       // );
       final imageWidget = SizedBox(
-        width: imageHeight,
+        width: imageWidth,
         height: imageHeight,
         child: Obx(() => Texture(textureId: widget.textureId.value)),
       );
@@ -521,13 +521,22 @@ class _ImagePaintState extends State<ImagePaint> {
       //   size: Size(c.size.width, c.size.height),
       //   painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s),
       // );
-      final imageWidget = Center(
+      if (c.size.width > 0 && c.size.height > 0) {
-        child: AspectRatio(
+        final imageWidget = Stack(
-          aspectRatio: c.size.width / c.size.height,
+          children: [
-          child: Obx(() =>  Texture(textureId: widget.textureId.value)),
+            Positioned(
-        ),
+              left: c.x,
+              top: c.y,
+              width: c.getDisplayWidth() * s,
+              height: c.getDisplayHeight() * s,
+              child: Texture(textureId: widget.textureId.value),
+            )
+          ],
         );
         return mouseRegion(child: _buildListener(imageWidget));
+      } else {
+        return Container();
+      }
     }
   }
 

flutter/lib/models/model.dart

@@ -252,6 +252,8 @@ class FfiModel with ChangeNotifier {
       parent.target?.cursorModel.updateDisplayOrigin(_display.x, _display.y);
     }
 
+    _updateSessionWidthHeight(peerId, display.width, display.height);
+
     try {
       CurrentDisplayState.find(peerId).value = _pi.currentDisplay;
     } catch (e) {
@@ -367,6 +369,10 @@ class FfiModel with ChangeNotifier {
     });
   }
 
+  _updateSessionWidthHeight(String id, int width, int height) {
+    bind.sessionSetSize(id: id, width: display.width, height: display.height);
+  }
+
   /// Handle the peer info event based on [evt].
   handlePeerInfo(Map<String, dynamic> evt, String peerId) async {
     // recent peer updated by handle_peer_info(ui_session_interface.rs) --> handle_peer_info(client.rs) --> save_config(client.rs)
@@ -420,6 +426,7 @@ class FfiModel with ChangeNotifier {
       stateGlobal.displaysCount.value = _pi.displays.length;
       if (_pi.currentDisplay < _pi.displays.length) {
         _display = _pi.displays[_pi.currentDisplay];
+        _updateSessionWidthHeight(peerId, display.width, display.height);
       }
       if (displays.isNotEmpty) {
         parent.target?.dialogManager.showLoading(
@@ -485,19 +492,18 @@ class ImageModel with ChangeNotifier {
 
   WeakReference<FFI> parent;
 
-  final List<Function(String)> _callbacksOnFirstImage = [];
+  final List<Function(String)> callbacksOnFirstImage = [];
 
   ImageModel(this.parent);
 
-  addCallbackOnFirstImage(Function(String) cb) =>
+  addCallbackOnFirstImage(Function(String) cb) => callbacksOnFirstImage.add(cb);
-      _callbacksOnFirstImage.add(cb);
 
   onRgba(Uint8List rgba) {
     if (_waitForImage[id]!) {
       _waitForImage[id] = false;
       parent.target?.dialogManager.dismissAll();
       if (isDesktop) {
-        for (final cb in _callbacksOnFirstImage) {
+        for (final cb in callbacksOnFirstImage) {
           cb(id);
         }
       }
@@ -1445,16 +1451,27 @@ class FFI {
             debugPrint('json.decode fail1(): $e, ${message.field0}');
           }
         } else if (message is EventToUI_Rgba) {
+          if (Platform.isAndroid || Platform.isIOS) {
             // Fetch the image buffer from rust codes.
-          // final sz = platformFFI.getRgbaSize(id);
+            final sz = platformFFI.getRgbaSize(id);
-          // if (sz == null || sz == 0) {
+            if (sz == null || sz == 0) {
-          //   return;
+              return;
-          // }
+            }
-          // final rgba = platformFFI.getRgba(id, sz);
+            final rgba = platformFFI.getRgba(id, sz);
-          // if (rgba != null) {
+            if (rgba != null) {
-          //   imageModel.onRgba(rgba);
+              imageModel.onRgba(rgba);
-          // }
+            }
-          // imageModel.onRgba(rgba);
+          } else {
+            if (_waitForImage[id]!) {
+              _waitForImage[id] = false;
+              dialogManager.dismissAll();
+              for (final cb in imageModel.callbacksOnFirstImage) {
+                cb(id);
+              }
+              await canvasModel.updateViewStyle();
+              await canvasModel.updateScrollStyle();
+            }
+          }
         }
       }
       debugPrint('Exit session event loop');

src/flutter.rs

@@ -5,12 +5,13 @@ use crate::{
 };
 use flutter_rust_bridge::StreamSink;
 use hbb_common::{
-    bail, config::LocalConfig, get_version_number, message_proto::*, rendezvous_proto::ConnType,
+    bail, config::LocalConfig, get_version_number, libc::c_void, message_proto::*,
-    ResultType,
+    rendezvous_proto::ConnType, ResultType,
 };
-use libc::{c_void};
 use libloading::{Library, Symbol};
 use serde_json::json;
+
+#[cfg(any(target_os = "android", target_os = "ios"))]
 use std::sync::atomic::{AtomicBool, Ordering};
 use std::{
     collections::HashMap,
@@ -30,7 +31,7 @@ lazy_static::lazy_static! {
     pub static ref SESSIONS: RwLock<HashMap<String, Session<FlutterHandler>>> = Default::default();
     pub static ref GLOBAL_EVENT_STREAM: RwLock<HashMap<String, StreamSink<String>>> = Default::default(); // rust to dart event channel
     #[cfg(not(any(target_os = "ios", target_os = "android")))]
-    pub static ref TEXURE_RGBA_RENDERER_PLUGIN: Library = {
+    pub static ref TEXTURE_RGBA_RENDERER_PLUGIN: Library = {
         unsafe {
             #[cfg(target_os = "windows")]
             let lib = Library::new("texture_rgba_renderer_plugin.dll");
@@ -127,21 +128,26 @@ pub struct FlutterHandler {
     pub event_stream: Arc<RwLock<Option<StreamSink<EventToUI>>>>,
     // SAFETY: [rgba] is guarded by [rgba_valid], and it's safe to reach [rgba] with `rgba_valid == true`.
     // We must check the `rgba_valid` before reading [rgba].
+    #[cfg(any(target_os = "android", target_os = "ios"))]
     pub rgba: Arc<RwLock<Vec<u8>>>,
+    #[cfg(any(target_os = "android", target_os = "ios"))]
     pub rgba_valid: Arc<AtomicBool>,
-    pub renderer: Arc<RwLock<VideoRenderer>>,
+    #[cfg(not(any(target_os = "android", target_os = "ios")))]
-    peer_info: Arc<RwLock<PeerInfo>>
+    notify_rendered: Arc<RwLock<bool>>,
+    renderer: Arc<RwLock<VideoRenderer>>,
+    peer_info: Arc<RwLock<PeerInfo>>,
 }
 pub type FlutterRgbaRendererPluginOnRgba =
     unsafe extern "C" fn(texture_rgba: *mut c_void, buffer: *const u8, width: c_int, height: c_int);
 
 // Video Texture Renderer in Flutter
 #[derive(Clone)]
-pub struct VideoRenderer {
+struct VideoRenderer {
     // TextureRgba pointer in flutter native.
     ptr: usize,
     width: i32,
     height: i32,
+    data_len: usize,
     on_rgba_func: Symbol<'static, FlutterRgbaRendererPluginOnRgba>,
 }
 
@@ -152,7 +158,8 @@ impl Default for VideoRenderer {
                 ptr: 0,
                 width: 0,
                 height: 0,
-                on_rgba_func: TEXURE_RGBA_RENDERER_PLUGIN
+                data_len: 0,
+                on_rgba_func: TEXTURE_RGBA_RENDERER_PLUGIN
                     .get::<FlutterRgbaRendererPluginOnRgba>(b"FlutterRgbaRendererPluginOnRgba")
                     .expect("Symbol FlutterRgbaRendererPluginOnRgba not found."),
             }
@@ -161,24 +168,30 @@ impl Default for VideoRenderer {
 }
 
 impl VideoRenderer {
-    pub fn new(ptr: usize) -> Self {
+    #[inline]
-        Self {
-            ptr,
-            ..Default::default()
-        }
-    }
-
     pub fn set_size(&mut self, width: i32, height: i32) {
         self.width = width;
         self.height = height;
+        self.data_len = if width > 0 && height > 0 {
+            (width * height * 4) as usize
+        } else {
+            0
+        };
     }
 
-    pub fn on_rgba(&self, rgba: *const u8) {
+    pub fn on_rgba(&self, rgba: &Vec<u8>) {
-        if self.ptr == usize::default() {
+        if self.ptr == usize::default() || rgba.len() != self.data_len {
             return;
         }
         let func = self.on_rgba_func.clone();
-        unsafe {func(self.ptr as _, rgba, self.width as _, self.height as _)};
+        unsafe {
+            func(
+                self.ptr as _,
+                rgba.as_ptr() as _,
+                self.width as _,
+                self.height as _,
+            )
+        };
     }
 }
 
@@ -222,9 +235,16 @@ impl FlutterHandler {
         serde_json::ser::to_string(&msg_vec).unwrap_or("".to_owned())
     }
 
+    #[inline]
     pub fn register_texture(&mut self, ptr: usize) {
         self.renderer.write().unwrap().ptr = ptr;
     }
+
+    #[inline]
+    pub fn set_size(&mut self, width: i32, height: i32) {
+        *self.notify_rendered.write().unwrap() = false;
+        self.renderer.write().unwrap().set_size(width, height);
+    }
 }
 
 impl InvokeUiSession for FlutterHandler {
@@ -385,6 +405,7 @@ impl InvokeUiSession for FlutterHandler {
     fn adapt_size(&self) {}
 
     #[inline]
+    #[cfg(any(target_os = "android", target_os = "ios"))]
     fn on_rgba(&self, data: &mut Vec<u8>) {
         // If the current rgba is not fetched by flutter, i.e., is valid.
         // We give up sending a new event to flutter.
@@ -397,11 +418,18 @@ impl InvokeUiSession for FlutterHandler {
         if let Some(stream) = &*self.event_stream.read().unwrap() {
             stream.add(EventToUI::Rgba);
         }
+    }
+
+    #[inline]
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
-        {
+    fn on_rgba(&self, data: &mut Vec<u8>) {
-            self.renderer.read().unwrap()
+        self.renderer.read().unwrap().on_rgba(data);
-            .on_rgba(self.rgba.read().unwrap().as_ptr());
+        if *self.notify_rendered.read().unwrap() {
-            self.next_rgba();
+            return;
+        }
+        if let Some(stream) = &*self.event_stream.read().unwrap() {
+            stream.add(EventToUI::Rgba);
+            *self.notify_rendered.write().unwrap() = true;
         }
     }
 
@@ -417,8 +445,6 @@ impl InvokeUiSession for FlutterHandler {
         }
         let features = serde_json::ser::to_string(&features).unwrap_or("".to_owned());
         *self.peer_info.write().unwrap() = pi.clone();
-        let curr_display = &pi.displays[pi.current_display as usize];
-        self.renderer.write().unwrap().set_size(curr_display.width, curr_display.height);
         self.push_event(
             "peer_info",
             vec![
@@ -467,8 +493,6 @@ impl InvokeUiSession for FlutterHandler {
     }
 
     fn switch_display(&self, display: &SwitchDisplay) {
-        let curr_display = &self.peer_info.read().unwrap().displays[display.display as usize];
-        self.renderer.write().unwrap().set_size(curr_display.width, curr_display.height);
         self.push_event(
             "switch_display",
             vec![
@@ -526,6 +550,7 @@ impl InvokeUiSession for FlutterHandler {
 
     #[inline]
     fn get_rgba(&self) -> *const u8 {
+        #[cfg(any(target_os = "android", target_os = "ios"))]
         if self.rgba_valid.load(Ordering::Relaxed) {
             return self.rgba.read().unwrap().as_ptr();
         }
@@ -534,6 +559,7 @@ impl InvokeUiSession for FlutterHandler {
 
     #[inline]
     fn next_rgba(&self) {
+        #[cfg(any(target_os = "android", target_os = "ios"))]
         self.rgba_valid.store(false, Ordering::Relaxed);
     }
 }
@@ -793,8 +819,10 @@ pub fn set_cur_session_id(id: String) {
 }
 
 #[no_mangle]
-pub fn session_get_rgba_size(id: *const char) -> usize {
+pub fn session_get_rgba_size(_id: *const char) -> usize {
-    let id = unsafe { std::ffi::CStr::from_ptr(id as _) };
+    #[cfg(any(target_os = "android", target_os = "ios"))]
+    let id = unsafe { std::ffi::CStr::from_ptr(_id as _) };
+    #[cfg(any(target_os = "android", target_os = "ios"))]
     if let Ok(id) = id.to_str() {
         if let Some(session) = SESSIONS.write().unwrap().get_mut(id) {
             return session.rgba.read().unwrap().len();

src/flutter_ffi.rs

@@ -529,6 +529,12 @@ pub fn session_switch_sides(id: String) {
     }
 }
 
+pub fn session_set_size(id: String, width: i32, height: i32)  {
+    if let Some(session) = SESSIONS.write().unwrap().get_mut(&id) {
+        session.set_size(width, height);
+    }
+}
+
 pub fn main_get_sound_inputs() -> Vec<String> {
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     return get_sound_inputs();