Refact/multi window soft rendering (#8343)

* refact: multi_window_soft_rendering

Signed-off-by: fufesou <linlong1266@gmail.com>

* fix: window pos, potential wait for image

Signed-off-by: fufesou <linlong1266@gmail.com>

* comments

Signed-off-by: fufesou <linlong1266@gmail.com>

* remove debug print

Signed-off-by: fufesou <linlong1266@gmail.com>

* explicitly set rgba_data.size_got to false after init

Signed-off-by: fufesou <linlong1266@gmail.com>

* refact: multi window, merge images, render with texture

Signed-off-by: fufesou <linlong1266@gmail.com>

* revert, flutter.rs, rgba valid

Signed-off-by: fufesou <linlong1266@gmail.com>

* Add displays index before sending capture msg

Signed-off-by: fufesou <linlong1266@gmail.com>

* refact: multi window, soft rendering

Signed-off-by: fufesou <linlong1266@gmail.com>

* fix: build

Signed-off-by: fufesou <linlong1266@gmail.com>

---------

Signed-off-by: fufesou <linlong1266@gmail.com>

flutter/lib/common.dart

@@ -2924,10 +2924,10 @@ openMonitorInNewTabOrWindow(int i, String peerId, PeerInfo pi,
       kMainWindowId, kWindowEventOpenMonitorSession, jsonEncode(args));
 }
 
-setNewConnectWindowFrame(int windowId, String peerId, Rect? screenRect) async {
+setNewConnectWindowFrame(int windowId, String peerId, int? display, Rect? screenRect) async {
   if (screenRect == null) {
     await restoreWindowPosition(WindowType.RemoteDesktop,
-        windowId: windowId, peerId: peerId);
+        windowId: windowId, display: display, peerId: peerId);
   } else {
     await tryMoveToScreenAndSetFullscreen(screenRect);
   }

flutter/lib/common/widgets/setting_widgets.dart

@@ -234,12 +234,12 @@ List<(String, String)> otherDefaultSettings() {
     ('True color (4:4:4)', kOptionI444),
     ('Reverse mouse wheel', kKeyReverseMouseWheel),
     ('swap-left-right-mouse', kOptionSwapLeftRightMouse),
-    if (isDesktop && bind.mainGetUseTextureRender())
+    if (isDesktop)
       (
         'Show displays as individual windows',
         kKeyShowDisplaysAsIndividualWindows
       ),
-    if (isDesktop && bind.mainGetUseTextureRender())
+    if (isDesktop)
       (
         'Use all my displays for the remote session',
         kKeyUseAllMyDisplaysForTheRemoteSession

flutter/lib/common/widgets/toolbar.dart

@@ -586,7 +586,6 @@ Future<List<TToggleMenu>> toolbarDisplayToggle(
   if (pi.isSupportMultiDisplay &&
       PrivacyModeState.find(id).isEmpty &&
       pi.displaysCount.value > 1 &&
-      bind.mainGetUseTextureRender() &&
       bind.mainGetUserDefaultOption(key: kKeyShowMonitorsToolbar) == 'Y') {
     final value =
         bind.sessionGetDisplaysAsIndividualWindows(sessionId: ffi.sessionId) ==
@@ -602,9 +601,7 @@ Future<List<TToggleMenu>> toolbarDisplayToggle(
   }
 
   final isMultiScreens = !isWeb && (await getScreenRectList()).length > 1;
-  if (bind.mainGetUseTextureRender() &&
+  if (pi.isSupportMultiDisplay && isMultiScreens) {
-      pi.isSupportMultiDisplay &&
-      isMultiScreens) {
     final value = bind.sessionGetUseAllMyDisplaysForTheRemoteSession(
             sessionId: ffi.sessionId) ==
         'Y';

flutter/lib/desktop/pages/remote_page.dart

@@ -617,7 +617,8 @@ class _ImagePaintState extends State<ImagePaint> {
       final paintWidth = c.getDisplayWidth() * s;
       final paintHeight = c.getDisplayHeight() * s;
       final paintSize = Size(paintWidth, paintHeight);
-      final paintWidget = m.useTextureRender
+      final paintWidget =
+          m.useTextureRender || widget.ffi.ffiModel.pi.forceTextureRender
               ? _BuildPaintTextureRender(
                   c, s, Offset.zero, paintSize, isViewOriginal())
               : _buildScrollbarNonTextureRender(m, paintSize, s);
@@ -638,7 +639,8 @@ class _ImagePaintState extends State<ImagePaint> {
           ));
     } else {
       if (c.size.width > 0 && c.size.height > 0) {
-        final paintWidget = m.useTextureRender
+        final paintWidget =
+            m.useTextureRender || widget.ffi.ffiModel.pi.forceTextureRender
                 ? _BuildPaintTextureRender(
                     c,
                     s,
@@ -648,7 +650,7 @@ class _ImagePaintState extends State<ImagePaint> {
                     ),
                     c.size,
                     isViewOriginal())
-            : _buildScrollAuthNonTextureRender(m, c, s);
+                : _buildScrollAutoNonTextureRender(m, c, s);
         return mouseRegion(child: _buildListener(paintWidget));
       } else {
         return Container();
@@ -664,7 +666,7 @@ class _ImagePaintState extends State<ImagePaint> {
     );
   }
 
-  Widget _buildScrollAuthNonTextureRender(
+  Widget _buildScrollAutoNonTextureRender(
       ImageModel m, CanvasModel c, double s) {
     return CustomPaint(
       size: Size(c.size.width, c.size.height),

flutter/lib/desktop/pages/remote_tab_page.dart

@@ -420,7 +420,7 @@ class _ConnectionTabPageState extends State<ConnectionTabPage> {
           await WindowController.fromWindowId(windowId()).setFullscreen(false);
           stateGlobal.setFullscreen(false, procWnd: false);
         }
-        await setNewConnectWindowFrame(windowId(), id!, screenRect);
+        await setNewConnectWindowFrame(windowId(), id!, display, screenRect);
         Future.delayed(Duration(milliseconds: isWindows ? 100 : 0), () async {
           await windowOnTop(windowId());
         });

flutter/lib/desktop/widgets/remote_toolbar.dart

@@ -643,15 +643,12 @@ class _MonitorMenu extends StatelessWidget {
   }
 
   Widget buildMonitorSubmenuWidget(BuildContext context) {
-    final m = Provider.of<ImageModel>(context);
     return Column(
       mainAxisSize: MainAxisSize.min,
       children: [
         Row(children: buildMonitorList(context, false)),
-        supportIndividualWindows && m.useTextureRender ? Divider() : Offstage(),
+        supportIndividualWindows ? Divider() : Offstage(),
-        supportIndividualWindows && m.useTextureRender
+        supportIndividualWindows ? chooseDisplayBehavior() : Offstage(),
-            ? chooseDisplayBehavior()
-            : Offstage(),
       ],
     );
   }
@@ -737,10 +734,7 @@ class _MonitorMenu extends StatelessWidget {
     for (int i = 0; i < pi.displays.length; i++) {
       monitorList.add(buildMonitorButton(i));
     }
-    final m = Provider.of<ImageModel>(context);
+    if (supportIndividualWindows && pi.displays.length > 1) {
-    if (supportIndividualWindows &&
-        m.useTextureRender &&
-        pi.displays.length > 1) {
       monitorList.add(buildMonitorButton(kAllDisplayValue));
     }
     return monitorList;
@@ -824,7 +818,6 @@ class _MonitorMenu extends StatelessWidget {
     RxInt display = CurrentDisplayState.find(id);
     if (display.value != i) {
       final isChooseDisplayToOpenInNewWindow = pi.isSupportMultiDisplay &&
-          bind.mainGetUseTextureRender() &&
           bind.sessionGetDisplaysAsIndividualWindows(
                   sessionId: ffi.sessionId) ==
               'Y';

flutter/lib/models/model.dart

@@ -1187,10 +1187,11 @@ class ImageModel with ChangeNotifier {
 
   onRgba(int display, Uint8List rgba) {
     final pid = parent.target?.id;
+    final rect = parent.target?.ffiModel.pi.getDisplayRect(display);
     img.decodeImageFromPixels(
         rgba,
-        parent.target?.ffiModel.rect?.width.toInt() ?? 0,
+        rect?.width.toInt() ?? 0,
-        parent.target?.ffiModel.rect?.height.toInt() ?? 0,
+        rect?.height.toInt() ?? 0,
         isWeb ? ui.PixelFormat.rgba8888 : ui.PixelFormat.bgra8888,
         onPixelsCopied: () {
       // Unlock the rgba memory from rust codes.
@@ -2397,9 +2398,10 @@ class FFI {
       cursorModel.peerId = id;
     }
 
+    final isNewPeer = tabWindowId == null;
     // If tabWindowId != null, this session is a "tab -> window" one.
     // Else this session is a new one.
-    if (tabWindowId == null) {
+    if (isNewPeer) {
       // ignore: unused_local_variable
       final addRes = bind.sessionAddSync(
         sessionId: sessionId,
@@ -2424,14 +2426,25 @@ class FFI {
             'Unreachable, failed to add existed session to $id, $addRes');
         return;
       }
-      bind.sessionTryAddDisplay(
-          sessionId: sessionId, displays: Int32List.fromList(displays));
       ffiModel.pi.currentDisplay = display;
     }
     if (isDesktop && connType == ConnType.defaultConn) {
       textureModel.updateCurrentDisplay(display ?? 0);
     }
-    final stream = bind.sessionStart(sessionId: sessionId, id: id);
+
+    // CAUTION: `sessionStart()` and `sessionStartWithDisplays()` are an async functions.
+    // Though the stream is returned immediately, the stream may not be ready.
+    // Any operations that depend on the stream should be carefully handled.
+    late final Stream<EventToUI> stream;
+    if (isNewPeer || display == null || displays == null) {
+      stream = bind.sessionStart(sessionId: sessionId, id: id);
+    } else {
+      // We have to put displays in `sessionStart()` to make sure the stream is ready
+      // and then the displays' capturing requests can be sent.
+      stream = bind.sessionStartWithDisplays(
+          sessionId: sessionId, id: id, displays: Int32List.fromList(displays));
+    }
+
     if (isWeb) {
       platformFFI.setRgbaCallback((int display, Uint8List data) {
         onEvent2UIRgba();
@@ -2442,14 +2455,6 @@ class FFI {
 
     final cb = ffiModel.startEventListener(sessionId, id);
 
-    // Force refresh displays.
-    // The controlled side may not refresh the image when the (peer,display) is already subscribed.
-    if (displays != null) {
-      for (final display in displays) {
-        bind.sessionRefresh(sessionId: sessionId, display: display);
-      }
-    }
-
     imageModel.updateUserTextureRender();
     final hasGpuTextureRender = bind.mainHasGpuTextureRender();
     final SimpleWrapper<bool> isToNewWindowNotified = SimpleWrapper(false);
@@ -2500,8 +2505,8 @@ class FFI {
           }
         } else if (message is EventToUI_Rgba) {
           final display = message.field0;
-          if (imageModel.useTextureRender) {
+          if (imageModel.useTextureRender || ffiModel.pi.forceTextureRender) {
-            debugPrint("EventToUI_Rgba display:$display");
+            //debugPrint("EventToUI_Rgba display:$display");
             textureModel.setTextureType(display: display, gpuTexture: false);
             onEvent2UIRgba();
           } else {
@@ -2692,6 +2697,7 @@ class PeerInfo with ChangeNotifier {
 
   bool get isSupportMultiDisplay =>
       (isDesktop || isWebDesktop) && isSupportMultiUiSession;
+  bool get forceTextureRender => currentDisplay == kAllDisplayValue;
 
   bool get cursorEmbedded => tryGetDisplay()?.cursorEmbedded ?? false;
 
@@ -2700,30 +2706,32 @@ class PeerInfo with ChangeNotifier {
   bool get isAmyuniIdd =>
       platformAdditions[kPlatformAdditionsIddImpl] == 'amyuni_idd';
 
-  Display? tryGetDisplay() {
+  Display? tryGetDisplay({int? display}) {
     if (displays.isEmpty) {
       return null;
     }
-    if (currentDisplay == kAllDisplayValue) {
+    display ??= currentDisplay;
+    if (display == kAllDisplayValue) {
       return displays[0];
     } else {
-      if (currentDisplay > 0 && currentDisplay < displays.length) {
+      if (display > 0 && display < displays.length) {
-        return displays[currentDisplay];
+        return displays[display];
       } else {
         return displays[0];
       }
     }
   }
 
-  Display? tryGetDisplayIfNotAllDisplay() {
+  Display? tryGetDisplayIfNotAllDisplay({int? display}) {
     if (displays.isEmpty) {
       return null;
     }
-    if (currentDisplay == kAllDisplayValue) {
+    display ??= currentDisplay;
+    if (display == kAllDisplayValue) {
       return null;
     }
-    if (currentDisplay >= 0 && currentDisplay < displays.length) {
+    if (display >= 0 && display < displays.length) {
-      return displays[currentDisplay];
+      return displays[display];
     } else {
       return null;
     }
@@ -2747,6 +2755,12 @@ class PeerInfo with ChangeNotifier {
     }
     return 1.0;
   }
+
+  Rect? getDisplayRect(int display) {
+    final d = tryGetDisplayIfNotAllDisplay(display: display);
+    if (d == null) return null;
+    return Rect.fromLTWH(d.x, d.y, d.width.toDouble(), d.height.toDouble());
+  }
 }
 
 const canvasKey = 'canvas';

flutter/lib/web/bridge.dart

@@ -63,11 +63,6 @@ class RustdeskImpl {
     return '';
   }
 
-  void sessionTryAddDisplay(
-      {required UuidValue sessionId,
-      required Int32List displays,
-      dynamic hint}) {}
-
   String sessionAddSync(
       {required UuidValue sessionId,
       required String id,
@@ -94,6 +89,14 @@ class RustdeskImpl {
     return Stream.empty();
   }
 
+  Stream<EventToUI> sessionStartWithDisplays(
+      {required UuidValue sessionId,
+      required String id,
+      required Int32List displays,
+      dynamic hint}) {
+    throw UnimplementedError();
+  }
+
   Future<bool?> sessionGetRemember(
       {required UuidValue sessionId, dynamic hint}) {
     return Future(

src/flutter.rs

@@ -408,6 +408,25 @@ impl VideoRenderer {
         }
     }
 
+    fn add_displays(&self, displays: &[i32]) {
+        let mut sessions_lock = self.map_display_sessions.write().unwrap();
+        for display in displays {
+            let d = *display as usize;
+            if !sessions_lock.contains_key(&d) {
+                sessions_lock.insert(
+                    d,
+                    DisplaySessionInfo {
+                        texture_rgba_ptr: 0,
+                        size: (0, 0),
+                        #[cfg(feature = "vram")]
+                        gpu_output_ptr: usize::default(),
+                        notify_render_type: None,
+                    },
+                );
+            }
+        }
+    }
+
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     pub fn on_rgba(&self, display: usize, rgba: &scrap::ImageRgb) -> bool {
         let mut write_lock = self.map_display_sessions.write().unwrap();
@@ -768,9 +787,9 @@ impl InvokeUiSession for FlutterHandler {
     #[inline]
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
     fn on_rgba(&self, display: usize, rgba: &mut scrap::ImageRgb) {
-        if self.use_texture_render.load(Ordering::Relaxed) {
+        let use_texture_render = self.use_texture_render.load(Ordering::Relaxed);
-            self.on_rgba_flutter_texture_render(display, rgba);
+        self.on_rgba_flutter_texture_render(use_texture_render, display, rgba);
-        } else {
+        if !use_texture_render {
             self.on_rgba_soft_render(display, rgba);
         }
     }
@@ -1039,19 +1058,39 @@ impl FlutterHandler {
         }
         drop(rgba_write_lock);
 
-        // Non-texture-render UI does not support multiple displays in the one UI session.
-        // It's Ok to notify each session for now.
         for h in self.session_handlers.read().unwrap().values() {
+            // `map_display_sessions` stores the display indices that are used by the video renderer.
+            let map_display_sessions = h.renderer.map_display_sessions.read().unwrap();
+            // The soft renderer does not support multi ui session for now.
+            if map_display_sessions.len() > 1 {
+                continue;
+            }
+            if h.renderer
+                .map_display_sessions
+                .read()
+                .unwrap()
+                .contains_key(&display)
+            {
+                if map_display_sessions.contains_key(&display) {
                     if let Some(stream) = &h.event_stream {
                         stream.add(EventToUI::Rgba(display));
                     }
                 }
             }
+        }
+    }
 
     #[inline]
     #[cfg(not(any(target_os = "android", target_os = "ios")))]
-    fn on_rgba_flutter_texture_render(&self, display: usize, rgba: &mut scrap::ImageRgb) {
+    fn on_rgba_flutter_texture_render(
+        &self,
+        use_texture_render: bool,
+        display: usize,
+        rgba: &mut scrap::ImageRgb,
+    ) {
         for (_, session) in self.session_handlers.read().unwrap().iter() {
+            if use_texture_render || session.renderer.map_display_sessions.read().unwrap().len() > 1
+            {
                 if session.renderer.on_rgba(display, rgba) {
                     if let Some(stream) = &session.event_stream {
                         stream.add(EventToUI::Rgba(display));
@@ -1060,6 +1099,7 @@ impl FlutterHandler {
             }
         }
     }
+}
 
 // This function is only used for the default connection session.
 pub fn session_add_existed(peer_id: String, session_id: SessionID) -> ResultType<()> {
@@ -1522,6 +1562,21 @@ pub fn session_register_gpu_texture(_session_id: SessionID, _display: usize, _ou
     }
 }
 
+pub fn session_add_displays(session_id: &SessionID, displays: &[i32]) {
+    for s in sessions::get_sessions() {
+        if let Some(h) = s
+            .ui_handler
+            .session_handlers
+            .read()
+            .unwrap()
+            .get(session_id)
+        {
+            h.renderer.add_displays(displays);
+            break;
+        }
+    }
+}
+
 #[inline]
 #[cfg(not(feature = "vram"))]
 pub fn get_adapter_luid() -> Option<i64> {

src/flutter_ffi.rs

@@ -110,13 +110,6 @@ pub fn session_add_existed_sync(id: String, session_id: SessionID) -> SyncReturn
     }
 }
 
-pub fn session_try_add_display(session_id: SessionID, displays: Vec<i32>) -> SyncReturn<()> {
-    if let Some(session) = sessions::get_session_by_session_id(&session_id) {
-        session.capture_displays(displays, vec![], vec![]);
-    }
-    SyncReturn(())
-}
-
 pub fn session_add_sync(
     session_id: SessionID,
     id: String,
@@ -153,6 +146,27 @@ pub fn session_start(
     session_start_(&session_id, &id, events2ui)
 }
 
+pub fn session_start_with_displays(
+    events2ui: StreamSink<EventToUI>,
+    session_id: SessionID,
+    id: String,
+    displays: Vec<i32>,
+) -> ResultType<()> {
+    session_start_(&session_id, &id, events2ui)?;
+
+    // `session_add_displays` is used for software rendering, multi-ui sessions.
+    // We need to add displays to the session first, then capture the displays.
+    // Otherwise, the session may not send video frames to the flutter side.
+    super::flutter::session_add_displays(&session_id, &displays);
+    if let Some(session) = sessions::get_session_by_session_id(&session_id) {
+        session.capture_displays(displays.clone(), vec![], vec![]);
+        for display in displays {
+            session.refresh_video(display as _);
+        }
+    }
+    Ok(())
+}
+
 pub fn session_get_remember(session_id: SessionID) -> Option<bool> {
     if let Some(session) = sessions::get_session_by_session_id(&session_id) {
         Some(session.get_remember())