From 5acedecf0c09546ec368ca22fa7367ec7b9c0ae5 Mon Sep 17 00:00:00 2001 From: fufesou Date: Tue, 21 Feb 2023 21:56:46 +0800 Subject: [PATCH] texture paint Signed-off-by: fufesou --- flutter/lib/desktop/pages/remote_page.dart | 33 ++++++--- flutter/lib/models/model.dart | 45 +++++++---- src/flutter.rs | 86 ++++++++++++++-------- src/flutter_ffi.rs | 6 ++ 4 files changed, 115 insertions(+), 55 deletions(-) diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index df9874172..4a2f5c0e8 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -126,9 +126,9 @@ class _RemotePageState extends State // Register texture. _textureId.value = -1; textureRenderer.createTexture(_textureKey).then((id) async { + debugPrint("id: $id, texture_key: $_textureKey"); if (id != -1) { final ptr = await textureRenderer.getTexturePtr(_textureKey); - debugPrint("id: $id, texture_key: $_textureKey"); platformFFI.registerTexture(widget.id, ptr); _textureId.value = id; } @@ -197,6 +197,8 @@ class _RemotePageState extends State @override void dispose() { debugPrint("REMOTE PAGE dispose ${widget.id}"); + platformFFI.registerTexture(widget.id, 0); + textureRenderer.closeTexture(_textureKey); // ensure we leave this session, this is a double check bind.sessionEnterOrLeave(id: widget.id, enter: false); DesktopMultiWindow.removeListener(this); @@ -212,7 +214,6 @@ class _RemotePageState extends State Wakelock.disable(); } Get.delete(tag: widget.id); - textureRenderer.closeTexture(_textureKey); super.dispose(); _removeStates(widget.id); } @@ -484,15 +485,14 @@ class _ImagePaintState extends State { final imageWidth = c.getDisplayWidth() * s; final imageHeight = c.getDisplayHeight() * s; final imageSize = Size(imageWidth, imageHeight); - print("width: $imageWidth/$imageHeight"); // final imageWidget = CustomPaint( // size: imageSize, // painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s), // ); final imageWidget = SizedBox( - width: imageHeight, + width: imageWidth, height: imageHeight, - child: Obx(() => Texture(textureId: widget.textureId.value)), + child: Obx(() => Texture(textureId: widget.textureId.value)), ); return NotificationListener( @@ -521,13 +521,22 @@ class _ImagePaintState extends State { // size: Size(c.size.width, c.size.height), // painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s), // ); - final imageWidget = Center( - child: AspectRatio( - aspectRatio: c.size.width / c.size.height, - child: Obx(() => Texture(textureId: widget.textureId.value)), - ), - ); - return mouseRegion(child: _buildListener(imageWidget)); + if (c.size.width > 0 && c.size.height > 0) { + final imageWidget = Stack( + children: [ + Positioned( + left: c.x, + top: c.y, + width: c.getDisplayWidth() * s, + height: c.getDisplayHeight() * s, + child: Texture(textureId: widget.textureId.value), + ) + ], + ); + return mouseRegion(child: _buildListener(imageWidget)); + } else { + return Container(); + } } } diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index 0b6f14636..a38db2a90 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -252,6 +252,8 @@ class FfiModel with ChangeNotifier { parent.target?.cursorModel.updateDisplayOrigin(_display.x, _display.y); } + _updateSessionWidthHeight(peerId, display.width, display.height); + try { CurrentDisplayState.find(peerId).value = _pi.currentDisplay; } catch (e) { @@ -367,6 +369,10 @@ class FfiModel with ChangeNotifier { }); } + _updateSessionWidthHeight(String id, int width, int height) { + bind.sessionSetSize(id: id, width: display.width, height: display.height); + } + /// Handle the peer info event based on [evt]. handlePeerInfo(Map evt, String peerId) async { // recent peer updated by handle_peer_info(ui_session_interface.rs) --> handle_peer_info(client.rs) --> save_config(client.rs) @@ -420,6 +426,7 @@ class FfiModel with ChangeNotifier { stateGlobal.displaysCount.value = _pi.displays.length; if (_pi.currentDisplay < _pi.displays.length) { _display = _pi.displays[_pi.currentDisplay]; + _updateSessionWidthHeight(peerId, display.width, display.height); } if (displays.isNotEmpty) { parent.target?.dialogManager.showLoading( @@ -485,19 +492,18 @@ class ImageModel with ChangeNotifier { WeakReference parent; - final List _callbacksOnFirstImage = []; + final List callbacksOnFirstImage = []; ImageModel(this.parent); - addCallbackOnFirstImage(Function(String) cb) => - _callbacksOnFirstImage.add(cb); + addCallbackOnFirstImage(Function(String) cb) => callbacksOnFirstImage.add(cb); onRgba(Uint8List rgba) { if (_waitForImage[id]!) { _waitForImage[id] = false; parent.target?.dialogManager.dismissAll(); if (isDesktop) { - for (final cb in _callbacksOnFirstImage) { + for (final cb in callbacksOnFirstImage) { cb(id); } } @@ -1445,16 +1451,27 @@ class FFI { debugPrint('json.decode fail1(): $e, ${message.field0}'); } } else if (message is EventToUI_Rgba) { - // Fetch the image buffer from rust codes. - // final sz = platformFFI.getRgbaSize(id); - // if (sz == null || sz == 0) { - // return; - // } - // final rgba = platformFFI.getRgba(id, sz); - // if (rgba != null) { - // imageModel.onRgba(rgba); - // } - // imageModel.onRgba(rgba); + if (Platform.isAndroid || Platform.isIOS) { + // Fetch the image buffer from rust codes. + final sz = platformFFI.getRgbaSize(id); + if (sz == null || sz == 0) { + return; + } + final rgba = platformFFI.getRgba(id, sz); + if (rgba != null) { + imageModel.onRgba(rgba); + } + } else { + if (_waitForImage[id]!) { + _waitForImage[id] = false; + dialogManager.dismissAll(); + for (final cb in imageModel.callbacksOnFirstImage) { + cb(id); + } + await canvasModel.updateViewStyle(); + await canvasModel.updateScrollStyle(); + } + } } } debugPrint('Exit session event loop'); diff --git a/src/flutter.rs b/src/flutter.rs index f5d764e66..a5689bce6 100644 --- a/src/flutter.rs +++ b/src/flutter.rs @@ -5,12 +5,13 @@ use crate::{ }; use flutter_rust_bridge::StreamSink; use hbb_common::{ - bail, config::LocalConfig, get_version_number, message_proto::*, rendezvous_proto::ConnType, - ResultType, + bail, config::LocalConfig, get_version_number, libc::c_void, message_proto::*, + rendezvous_proto::ConnType, ResultType, }; -use libc::{c_void}; use libloading::{Library, Symbol}; use serde_json::json; + +#[cfg(any(target_os = "android", target_os = "ios"))] use std::sync::atomic::{AtomicBool, Ordering}; use std::{ collections::HashMap, @@ -30,7 +31,7 @@ lazy_static::lazy_static! { pub static ref SESSIONS: RwLock>> = Default::default(); pub static ref GLOBAL_EVENT_STREAM: RwLock>> = Default::default(); // rust to dart event channel #[cfg(not(any(target_os = "ios", target_os = "android")))] - pub static ref TEXURE_RGBA_RENDERER_PLUGIN: Library = { + pub static ref TEXTURE_RGBA_RENDERER_PLUGIN: Library = { unsafe { #[cfg(target_os = "windows")] let lib = Library::new("texture_rgba_renderer_plugin.dll"); @@ -127,21 +128,26 @@ pub struct FlutterHandler { pub event_stream: Arc>>>, // SAFETY: [rgba] is guarded by [rgba_valid], and it's safe to reach [rgba] with `rgba_valid == true`. // We must check the `rgba_valid` before reading [rgba]. + #[cfg(any(target_os = "android", target_os = "ios"))] pub rgba: Arc>>, + #[cfg(any(target_os = "android", target_os = "ios"))] pub rgba_valid: Arc, - pub renderer: Arc>, - peer_info: Arc> + #[cfg(not(any(target_os = "android", target_os = "ios")))] + notify_rendered: Arc>, + renderer: Arc>, + peer_info: Arc>, } pub type FlutterRgbaRendererPluginOnRgba = unsafe extern "C" fn(texture_rgba: *mut c_void, buffer: *const u8, width: c_int, height: c_int); // Video Texture Renderer in Flutter #[derive(Clone)] -pub struct VideoRenderer { +struct VideoRenderer { // TextureRgba pointer in flutter native. ptr: usize, width: i32, height: i32, + data_len: usize, on_rgba_func: Symbol<'static, FlutterRgbaRendererPluginOnRgba>, } @@ -152,7 +158,8 @@ impl Default for VideoRenderer { ptr: 0, width: 0, height: 0, - on_rgba_func: TEXURE_RGBA_RENDERER_PLUGIN + data_len: 0, + on_rgba_func: TEXTURE_RGBA_RENDERER_PLUGIN .get::(b"FlutterRgbaRendererPluginOnRgba") .expect("Symbol FlutterRgbaRendererPluginOnRgba not found."), } @@ -161,24 +168,30 @@ impl Default for VideoRenderer { } impl VideoRenderer { - pub fn new(ptr: usize) -> Self { - Self { - ptr, - ..Default::default() - } - } - + #[inline] pub fn set_size(&mut self, width: i32, height: i32) { self.width = width; self.height = height; + self.data_len = if width > 0 && height > 0 { + (width * height * 4) as usize + } else { + 0 + }; } - pub fn on_rgba(&self, rgba: *const u8) { - if self.ptr == usize::default() { + pub fn on_rgba(&self, rgba: &Vec) { + if self.ptr == usize::default() || rgba.len() != self.data_len { return; } let func = self.on_rgba_func.clone(); - unsafe {func(self.ptr as _, rgba, self.width as _, self.height as _)}; + unsafe { + func( + self.ptr as _, + rgba.as_ptr() as _, + self.width as _, + self.height as _, + ) + }; } } @@ -222,9 +235,16 @@ impl FlutterHandler { serde_json::ser::to_string(&msg_vec).unwrap_or("".to_owned()) } + #[inline] pub fn register_texture(&mut self, ptr: usize) { self.renderer.write().unwrap().ptr = ptr; } + + #[inline] + pub fn set_size(&mut self, width: i32, height: i32) { + *self.notify_rendered.write().unwrap() = false; + self.renderer.write().unwrap().set_size(width, height); + } } impl InvokeUiSession for FlutterHandler { @@ -385,6 +405,7 @@ impl InvokeUiSession for FlutterHandler { fn adapt_size(&self) {} #[inline] + #[cfg(any(target_os = "android", target_os = "ios"))] fn on_rgba(&self, data: &mut Vec) { // If the current rgba is not fetched by flutter, i.e., is valid. // We give up sending a new event to flutter. @@ -397,11 +418,18 @@ impl InvokeUiSession for FlutterHandler { if let Some(stream) = &*self.event_stream.read().unwrap() { stream.add(EventToUI::Rgba); } - #[cfg(not(any(target_os = "android", target_os = "ios")))] - { - self.renderer.read().unwrap() - .on_rgba(self.rgba.read().unwrap().as_ptr()); - self.next_rgba(); + } + + #[inline] + #[cfg(not(any(target_os = "android", target_os = "ios")))] + fn on_rgba(&self, data: &mut Vec) { + self.renderer.read().unwrap().on_rgba(data); + if *self.notify_rendered.read().unwrap() { + return; + } + if let Some(stream) = &*self.event_stream.read().unwrap() { + stream.add(EventToUI::Rgba); + *self.notify_rendered.write().unwrap() = true; } } @@ -417,8 +445,6 @@ impl InvokeUiSession for FlutterHandler { } let features = serde_json::ser::to_string(&features).unwrap_or("".to_owned()); *self.peer_info.write().unwrap() = pi.clone(); - let curr_display = &pi.displays[pi.current_display as usize]; - self.renderer.write().unwrap().set_size(curr_display.width, curr_display.height); self.push_event( "peer_info", vec![ @@ -467,8 +493,6 @@ impl InvokeUiSession for FlutterHandler { } fn switch_display(&self, display: &SwitchDisplay) { - let curr_display = &self.peer_info.read().unwrap().displays[display.display as usize]; - self.renderer.write().unwrap().set_size(curr_display.width, curr_display.height); self.push_event( "switch_display", vec![ @@ -526,6 +550,7 @@ impl InvokeUiSession for FlutterHandler { #[inline] fn get_rgba(&self) -> *const u8 { + #[cfg(any(target_os = "android", target_os = "ios"))] if self.rgba_valid.load(Ordering::Relaxed) { return self.rgba.read().unwrap().as_ptr(); } @@ -534,6 +559,7 @@ impl InvokeUiSession for FlutterHandler { #[inline] fn next_rgba(&self) { + #[cfg(any(target_os = "android", target_os = "ios"))] self.rgba_valid.store(false, Ordering::Relaxed); } } @@ -793,8 +819,10 @@ pub fn set_cur_session_id(id: String) { } #[no_mangle] -pub fn session_get_rgba_size(id: *const char) -> usize { - let id = unsafe { std::ffi::CStr::from_ptr(id as _) }; +pub fn session_get_rgba_size(_id: *const char) -> usize { + #[cfg(any(target_os = "android", target_os = "ios"))] + let id = unsafe { std::ffi::CStr::from_ptr(_id as _) }; + #[cfg(any(target_os = "android", target_os = "ios"))] if let Ok(id) = id.to_str() { if let Some(session) = SESSIONS.write().unwrap().get_mut(id) { return session.rgba.read().unwrap().len(); diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 7eeb96b5c..c55866dbe 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -529,6 +529,12 @@ pub fn session_switch_sides(id: String) { } } +pub fn session_set_size(id: String, width: i32, height: i32) { + if let Some(session) = SESSIONS.write().unwrap().get_mut(&id) { + session.set_size(width, height); + } +} + pub fn main_get_sound_inputs() -> Vec { #[cfg(not(any(target_os = "android", target_os = "ios")))] return get_sound_inputs();