mobile & web rgba stream

flutter/lib/models/model.dart

@@ -17,12 +17,11 @@ import '../widgets/overlay.dart';
 import 'native_model.dart' if (dart.library.html) 'web_model.dart';
 
 typedef HandleMsgBox = void Function(Map<String, dynamic> evt, String id);
+bool _waitForImage = false;
 
 class FfiModel with ChangeNotifier {
   PeerInfo _pi = PeerInfo();
   Display _display = Display();
-  var _decoding = false;
-  bool _waitForImage = false;
   var _inputBlocked = false;
   final _permissions = Map<String, bool>();
   bool? _secure;
@@ -122,7 +121,6 @@ class FfiModel with ChangeNotifier {
 
   void updateEventListener(String peerId) {
     final void Function(Map<String, dynamic>) cb = (evt) {
-      var pos;
       var name = evt['name'];
       if (name == 'msgbox') {
         handleMsgBox(evt, peerId);
@@ -138,7 +136,7 @@ class FfiModel with ChangeNotifier {
       } else if (name == 'cursor_id') {
         FFI.cursorModel.updateCursorId(evt);
       } else if (name == 'cursor_position') {
-        pos = evt;
+        FFI.cursorModel.updateCursorPosition(evt);
       } else if (name == 'clipboard') {
         Clipboard.setData(ClipboardData(text: evt['content']));
       } else if (name == 'permission') {
@@ -165,31 +163,6 @@ class FfiModel with ChangeNotifier {
       } else if (name == 'on_client_remove') {
         FFI.serverModel.onClientRemove(evt);
       }
-      if (pos != null) FFI.cursorModel.updateCursorPosition(pos);
-      if (!_decoding) {
-        var rgba = PlatformFFI.getRgba();
-        if (rgba != null) {
-          if (_waitForImage) {
-            _waitForImage = false;
-            SmartDialog.dismiss();
-          }
-          _decoding = true;
-          final pid = FFI.id;
-          ui.decodeImageFromPixels(rgba, _display.width, _display.height,
-              isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888,
-              (image) {
-            PlatformFFI.clearRgbaFrame();
-            _decoding = false;
-            if (FFI.id != pid) return;
-            try {
-              // my throw exception, because the listener maybe already dispose
-              FFI.imageModel.update(image);
-            } catch (e) {
-              print('update image: $e');
-            }
-          });
-        }
-      }
     };
     PlatformFFI.setEventCallback(cb);
   }
@@ -284,6 +257,29 @@ class ImageModel with ChangeNotifier {
 
   ui.Image? get image => _image;
 
+  ImageModel() {
+    PlatformFFI.setRgbaCallback((rgba) {
+      if (_waitForImage) {
+        _waitForImage = false;
+        SmartDialog.dismiss();
+      }
+      final pid = FFI.id;
+      ui.decodeImageFromPixels(
+          rgba,
+          FFI.ffiModel.display.width,
+          FFI.ffiModel.display.height,
+          isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888, (image) {
+        if (FFI.id != pid) return;
+        try {
+          // my throw exception, because the listener maybe already dispose
+          FFI.imageModel.update(image);
+        } catch (e) {
+          print('update image: $e');
+        }
+      });
+    });
+  }
+
   void update(ui.Image? image) {
     if (_image == null && image != null) {
       if (isDesktop) {

flutter/lib/models/native_model.dart

@@ -19,8 +19,6 @@ class RgbaFrame extends Struct {
 
 typedef F2 = Pointer<Utf8> Function(Pointer<Utf8>, Pointer<Utf8>);
 typedef F3 = void Function(Pointer<Utf8>, Pointer<Utf8>);
-typedef F4 = void Function(Pointer<RgbaFrame>);
-typedef F5 = Pointer<RgbaFrame> Function();
 
 class PlatformFFI {
   static Pointer<RgbaFrame>? _lastRgbaFrame;
@@ -28,23 +26,8 @@ class PlatformFFI {
   static String _homeDir = '';
   static F2? _getByName;
   static F3? _setByName;
-  static F4? _freeRgba;
-  static F5? _getRgba;
   static void Function(Map<String, dynamic>)? _eventCallback;
-
+  static void Function(Uint8List)? _rgbaCallback;
-  static void clearRgbaFrame() {
-    if (_lastRgbaFrame != null &&
-        _lastRgbaFrame != nullptr &&
-        _freeRgba != null) _freeRgba!(_lastRgbaFrame!);
-  }
-
-  static Uint8List? getRgba() {
-    if (_getRgba == null) return null;
-    _lastRgbaFrame = _getRgba!();
-    if (_lastRgbaFrame == null || _lastRgbaFrame == nullptr) return null;
-    final ref = _lastRgbaFrame!.ref;
-    return Uint8List.sublistView(ref.data.asTypedList(ref.len));
-  }
 
   static Future<String> getVersion() async {
     PackageInfo packageInfo = await PackageInfo.fromPlatform();
@@ -85,9 +68,6 @@ class PlatformFFI {
       _setByName =
           dylib.lookupFunction<Void Function(Pointer<Utf8>, Pointer<Utf8>), F3>(
               'set_by_name');
-      _freeRgba = dylib
-          .lookupFunction<Void Function(Pointer<RgbaFrame>), F4>('free_rgba');
-      _getRgba = dylib.lookupFunction<F5, F5>('get_rgba');
       _dir = (await getApplicationDocumentsDirectory()).path;
       _startListenEvent(RustdeskImpl(dylib));
       try {
@@ -119,23 +99,38 @@ class PlatformFFI {
     version = await getVersion();
   }
 
-  static void _startListenEvent(RustdeskImpl rustdeskImpl) async {
+  static void _startListenEvent(RustdeskImpl rustdeskImpl) {
-    await for (final message in rustdeskImpl.startEventStream()) {
+    () async {
-      if (_eventCallback != null) {
+      await for (final message in rustdeskImpl.startEventStream()) {
-        try {
+        if (_eventCallback != null) {
-          Map<String, dynamic> event = json.decode(message);
+          try {
-          _eventCallback!(event);
+            Map<String, dynamic> event = json.decode(message);
-        } catch (e) {
+            _eventCallback!(event);
-          print('json.decode fail(): $e');
+          } catch (e) {
+            print('json.decode fail(): $e');
+          }
         }
       }
-    }
+    }();
+    () async {
+      await for (final rgba in rustdeskImpl.startRgbaStream()) {
+        if (_rgbaCallback != null) {
+          _rgbaCallback!(rgba);
+        } else {
+          rgba.clear();
+        }
+      }
+    }();
   }
 
   static void setEventCallback(void Function(Map<String, dynamic>) fun) async {
     _eventCallback = fun;
   }
 
+  static void setRgbaCallback(void Function(Uint8List) fun) async {
+    _rgbaCallback = fun;
+  }
+
   static void startDesktopWebListener() {}
 
   static void stopDesktopWebListener() {}

flutter/lib/models/web_model.dart

@@ -10,12 +10,6 @@ final List<StreamSubscription<MouseEvent>> mouseListeners = [];
 final List<StreamSubscription<KeyboardEvent>> keyListeners = [];
 
 class PlatformFFI {
-  static void clearRgbaFrame() {}
-
-  static Uint8List? getRgba() {
-    return context.callMethod('getRgba');
-  }
-
   static String getByName(String name, [String arg = '']) {
     return context.callMethod('getByName', [name, arg]);
   }
@@ -31,7 +25,7 @@ class PlatformFFI {
     version = getByName('version');
   }
 
-  static void setEventCallback(void Function(Map<String, dynamic>) fun) async {
+  static void setEventCallback(void Function(Map<String, dynamic>) fun) {
     context["onGlobalEvent"] = (String message) {
       try {
         Map<String, dynamic> event = json.decode(message);
@@ -42,6 +36,14 @@ class PlatformFFI {
     };
   }
 
+  static void setRgbaCallback(void Function(Uint8List) fun) {
+    context["onRgba"] = (Uint8List? rgba) {
+      if (rgba != null) {
+        fun(rgba);
+      }
+    };
+  }
+
   static void startDesktopWebListener() {
     mouseListeners.add(
         window.document.onContextMenu.listen((evt) => evt.preventDefault()));

flutter/web/js/src/globals.js

@@ -6,14 +6,7 @@ import { checkIfRetry, version } from "./gen_js_from_hbb";
 import { initZstd, translate } from "./common";
 import PCMPlayer from "pcm-player";
 
-var currentFrame = undefined;
-
 window.curConn = undefined;
-window.getRgba = () => {
-  const tmp = currentFrame;
-  currentFrame = undefined;
-  return tmp || null;
-}
 window.isMobile = () => {
   return /(android|bb\d+|meego).+mobile|avantgo|bada\/|blackberry|blazer|compal|elaine|fennec|hiptop|iemobile|ip(hone|od)|ipad|iris|kindle|Android|Silk|lge |maemo|midp|mmp|netfront|opera m(ob|in)i|palm( os)?|phone|p(ixi|re)\/|plucker|pocket|psp|series(4|6)0|symbian|treo|up\.(browser|link)|vodafone|wap|windows (ce|phone)|xda|xiino/i.test(navigator.userAgent)
     || /1207|6310|6590|3gso|4thp|50[1-6]i|770s|802s|a wa|abac|ac(er|oo|s\-)|ai(ko|rn)|al(av|ca|co)|amoi|an(ex|ny|yw)|aptu|ar(ch|go)|as(te|us)|attw|au(di|\-m|r |s )|avan|be(ck|ll|nq)|bi(lb|rd)|bl(ac|az)|br(e|v)w|bumb|bw\-(n|u)|c55\/|capi|ccwa|cdm\-|cell|chtm|cldc|cmd\-|co(mp|nd)|craw|da(it|ll|ng)|dbte|dc\-s|devi|dica|dmob|do(c|p)o|ds(12|\-d)|el(49|ai)|em(l2|ul)|er(ic|k0)|esl8|ez([4-7]0|os|wa|ze)|fetc|fly(\-|_)|g1 u|g560|gene|gf\-5|g\-mo|go(\.w|od)|gr(ad|un)|haie|hcit|hd\-(m|p|t)|hei\-|hi(pt|ta)|hp( i|ip)|hs\-c|ht(c(\-| |_|a|g|p|s|t)|tp)|hu(aw|tc)|i\-(20|go|ma)|i230|iac( |\-|\/)|ibro|idea|ig01|ikom|im1k|inno|ipaq|iris|ja(t|v)a|jbro|jemu|jigs|kddi|keji|kgt( |\/)|klon|kpt |kwc\-|kyo(c|k)|le(no|xi)|lg( g|\/(k|l|u)|50|54|\-[a-w])|libw|lynx|m1\-w|m3ga|m50\/|ma(te|ui|xo)|mc(01|21|ca)|m\-cr|me(rc|ri)|mi(o8|oa|ts)|mmef|mo(01|02|bi|de|do|t(\-| |o|v)|zz)|mt(50|p1|v )|mwbp|mywa|n10[0-2]|n20[2-3]|n30(0|2)|n50(0|2|5)|n7(0(0|1)|10)|ne((c|m)\-|on|tf|wf|wg|wt)|nok(6|i)|nzph|o2im|op(ti|wv)|oran|owg1|p800|pan(a|d|t)|pdxg|pg(13|\-([1-8]|c))|phil|pire|pl(ay|uc)|pn\-2|po(ck|rt|se)|prox|psio|pt\-g|qa\-a|qc(07|12|21|32|60|\-[2-7]|i\-)|qtek|r380|r600|raks|rim9|ro(ve|zo)|s55\/|sa(ge|ma|mm|ms|ny|va)|sc(01|h\-|oo|p\-)|sdk\/|se(c(\-|0|1)|47|mc|nd|ri)|sgh\-|shar|sie(\-|m)|sk\-0|sl(45|id)|sm(al|ar|b3|it|t5)|so(ft|ny)|sp(01|h\-|v\-|v )|sy(01|mb)|t2(18|50)|t6(00|10|18)|ta(gt|lk)|tcl\-|tdg\-|tel(i|m)|tim\-|t\-mo|to(pl|sh)|ts(70|m\-|m3|m5)|tx\-9|up(\.b|g1|si)|utst|v400|v750|veri|vi(rg|te)|vk(40|5[0-3]|\-v)|vm40|voda|vulc|vx(52|53|60|61|70|80|81|83|85|98)|w3c(\-| )|webc|whit|wi(g |nc|nw)|wmlb|wonu|x700|yas\-|your|zeto|zte\-/i.test(navigator.userAgent.substr(0, 4));
@@ -81,7 +74,7 @@ export function draw(frame) {
     for (let i = 0; i < size; i += row) {
       flipPixels.set(pixels.subarray(i, i + row), end - i);
     }
-    currentFrame = flipPixels;
+    onRgba(flipPixels);
     testSpeed[1] += new Date().getTime() - tm0;
     testSpeed[0] += 1;
     if (testSpeed[0] > 30) {
@@ -116,7 +109,6 @@ export function getConn() {
 }
 
 export async function startConn(id) {
-  currentFrame = undefined;
   setByName('remote_id', id);
   await curConn.start(id);
 }
@@ -124,7 +116,6 @@ export async function startConn(id) {
 export function close() {
   getConn()?.close();
   setConn(undefined);
-  currentFrame = undefined;
 }
 
 export function newConn() {
@@ -339,7 +330,7 @@ export function playAudio(packet) {
 window.init = async () => {
   if (yuvWorker) {
     yuvWorker.onmessage = (e) => {
-      currentFrame = e.data;
+      onRgba(e.data);
     }
   }
   opusWorker.onmessage = (e) => {

src/mobile.rs

@@ -1,5 +1,5 @@
 use crate::client::*;
-use flutter_rust_bridge::StreamSink;
+use flutter_rust_bridge::{StreamSink, ZeroCopyBuffer};
 use hbb_common::{
     allow_err,
     compress::decompress,
@@ -24,7 +24,8 @@ use std::{
 
 lazy_static::lazy_static! {
     static ref SESSION: Arc<RwLock<Option<Session>>> = Default::default();
-    pub static ref EVENT_STREAM: RwLock<Option<StreamSink<String>>> = Default::default(); // rust to dart channel
+    pub static ref EVENT_STREAM: RwLock<Option<StreamSink<String>>> = Default::default(); // rust to dart event channel
+    pub static ref RGBA_STREAM: RwLock<Option<StreamSink<ZeroCopyBuffer<Vec<u8>>>>> = Default::default(); // rust to dart rgba (big u8 list) channel
 }
 
 #[derive(Clone, Default)]
@@ -33,7 +34,6 @@ pub struct Session {
     sender: Arc<RwLock<Option<mpsc::UnboundedSender<Data>>>>,
     lc: Arc<RwLock<LoginConfigHandler>>,
     events2ui: Arc<RwLock<VecDeque<String>>>,
-    rgba: Arc<RwLock<Option<Vec<u8>>>>,
 }
 
 impl Session {
@@ -89,14 +89,6 @@ impl Session {
         }
     }
 
-    pub fn rgba() -> Option<Vec<u8>> {
-        if let Some(session) = SESSION.read().unwrap().as_ref() {
-            session.rgba.write().unwrap().take()
-        } else {
-            None
-        }
-    }
-
     pub fn pop_event() -> Option<String> {
         if let Some(session) = SESSION.read().unwrap().as_ref() {
             session.events2ui.write().unwrap().pop_front()
@@ -607,8 +599,11 @@ impl Connection {
                     if !self.first_frame {
                         self.first_frame = true;
                     }
-                    if let Ok(true) = self.video_handler.handle_frame(vf) {
+                    if let (Ok(true), Some(s)) = (
-                        *self.session.rgba.write().unwrap() = Some(self.video_handler.rgb.clone());
+                        self.video_handler.handle_frame(vf),
+                        RGBA_STREAM.read().unwrap().as_ref(),
+                    ) {
+                        s.add(ZeroCopyBuffer(self.video_handler.rgb.clone()));
                     }
                 }
                 Some(message::Union::hash(hash)) => {

src/mobile_ffi.rs

@@ -1,7 +1,7 @@
 use crate::client::file_trait::FileManager;
 use crate::mobile::connection_manager::{self, get_clients_length, get_clients_state};
 use crate::mobile::{self, make_fd_to_json, Session};
-use flutter_rust_bridge::StreamSink;
+use flutter_rust_bridge::{StreamSink, ZeroCopyBuffer};
 use hbb_common::ResultType;
 use hbb_common::{
     config::{self, Config, LocalConfig, PeerConfig, ONLINE},
@@ -40,6 +40,11 @@ pub fn start_event_stream(s: StreamSink<String>) -> ResultType<()> {
     Ok(())
 }
 
+pub fn start_rgba_stream(s: StreamSink<ZeroCopyBuffer<Vec<u8>>>) -> ResultType<()> {
+    let _ = mobile::RGBA_STREAM.write().unwrap().insert(s);
+    Ok(())
+}
+
 #[no_mangle]
 unsafe extern "C" fn get_by_name(name: *const c_char, arg: *const c_char) -> *const c_char {
     let mut res = "".to_owned();
@@ -359,7 +364,7 @@ unsafe extern "C" fn set_by_name(name: *const c_char, value: *const c_char) {
                             m.get("file_num"),
                             m.get("need_override"),
                             m.get("remember"),
-                            m.get("is_upload")
+                            m.get("is_upload"),
                         ) {
                             Session::set_confirm_override_file(
                                 id.parse().unwrap_or(0),
@@ -507,44 +512,6 @@ unsafe extern "C" fn set_by_name(name: *const c_char, value: *const c_char) {
     }
 }
 
-#[repr(C)]
-struct RgbaFrame {
-    len: u32,
-    data: *mut u8,
-}
-
-#[no_mangle]
-unsafe extern "C" fn get_rgba() -> *mut RgbaFrame {
-    if let Some(mut vec) = Session::rgba() {
-        if vec.is_empty() {
-            return std::ptr::null_mut();
-        }
-        assert!(vec.len() == vec.capacity());
-        vec.shrink_to_fit();
-        let data = vec.as_mut_ptr();
-        let len = vec.len();
-        std::mem::forget(vec);
-        Box::into_raw(Box::new(RgbaFrame {
-            len: len as _,
-            data,
-        }))
-    } else {
-        std::ptr::null_mut()
-    }
-}
-
-#[no_mangle]
-extern "C" fn free_rgba(f: *mut RgbaFrame) {
-    if f.is_null() {
-        return;
-    }
-    unsafe {
-        let len = (*f).len as usize;
-        drop(Vec::from_raw_parts((*f).data, len, len));
-        Box::from_raw(f);
-    }
-}
-
 #[cfg(target_os = "android")]
 pub mod server_side {
     use hbb_common::{config::Config, log};