From 7a5bc864fa357c9b2729868ee525d5c56fdaf216 Mon Sep 17 00:00:00 2001 From: 21pages Date: Wed, 18 Oct 2023 22:39:28 +0800 Subject: [PATCH] fix client side record Signed-off-by: 21pages --- flutter/lib/common/widgets/toolbar.dart | 5 +- flutter/lib/desktop/pages/remote_page.dart | 3 +- .../lib/desktop/widgets/remote_toolbar.dart | 8 +- flutter/lib/mobile/pages/remote_page.dart | 4 +- flutter/lib/models/model.dart | 79 +++++++++++-------- libs/scrap/src/common/record.rs | 48 ++++++++--- src/client.rs | 8 +- src/server/connection.rs | 3 +- src/ui/header.tis | 14 +++- src/ui/remote.rs | 6 ++ 10 files changed, 119 insertions(+), 59 deletions(-) diff --git a/flutter/lib/common/widgets/toolbar.dart b/flutter/lib/common/widgets/toolbar.dart index 757a03fec..28b10785b 100644 --- a/flutter/lib/common/widgets/toolbar.dart +++ b/flutter/lib/common/widgets/toolbar.dart @@ -224,11 +224,8 @@ List toolbarControls(BuildContext context, String id, FFI ffi) { )); } // record - var codecFormat = ffi.qualityMonitorModel.data.codecFormat; if (!isDesktop && - (ffi.recordingModel.start || - (perms["recording"] != false && - (codecFormat == "VP8" || codecFormat == "VP9")))) { + (ffi.recordingModel.start || (perms["recording"] != false))) { v.add(TTextMenu( child: Row( children: [ diff --git a/flutter/lib/desktop/pages/remote_page.dart b/flutter/lib/desktop/pages/remote_page.dart index abbb8785d..cb9cf49c9 100644 --- a/flutter/lib/desktop/pages/remote_page.dart +++ b/flutter/lib/desktop/pages/remote_page.dart @@ -677,7 +677,8 @@ class _ImagePaintState extends State { } else { final key = cache.updateGetKey(scale); if (!cursor.cachedKeys.contains(key)) { - debugPrint("Register custom cursor with key $key (${cache.hotx},${cache.hoty})"); + debugPrint( + "Register custom cursor with key $key (${cache.hotx},${cache.hoty})"); // [Safety] // It's ok to call async registerCursor in current synchronous context, // because activating the cursor is also an async call and will always diff --git a/flutter/lib/desktop/widgets/remote_toolbar.dart b/flutter/lib/desktop/widgets/remote_toolbar.dart index 240add919..fd4fcc434 100644 --- a/flutter/lib/desktop/widgets/remote_toolbar.dart +++ b/flutter/lib/desktop/widgets/remote_toolbar.dart @@ -1370,11 +1370,12 @@ class _ResolutionsMenuState extends State<_ResolutionsMenu> { } for (final r in resolutions) { - if (r.width == _localResolution!.width && r.height == _localResolution!.height) { + if (r.width == _localResolution!.width && + r.height == _localResolution!.height) { return r; } } - + return null; } @@ -1652,7 +1653,8 @@ class _RecordMenu extends StatelessWidget { var ffi = Provider.of(context); var recordingModel = Provider.of(context); final visible = - recordingModel.start || ffi.permissions['recording'] != false; + (recordingModel.start || ffi.permissions['recording'] != false) && + ffi.pi.currentDisplay != kAllDisplayValue; if (!visible) return Offstage(); return _IconMenuButton( assetName: 'assets/rec.svg', diff --git a/flutter/lib/mobile/pages/remote_page.dart b/flutter/lib/mobile/pages/remote_page.dart index 249355012..ff71c9347 100644 --- a/flutter/lib/mobile/pages/remote_page.dart +++ b/flutter/lib/mobile/pages/remote_page.dart @@ -763,7 +763,9 @@ void showOptions( children.add(InkWell( onTap: () { if (i == cur) return; - bind.sessionSwitchDisplay(sessionId: gFFI.sessionId, value: Int32List.fromList([i])); + gFFI.recordingModel.onClose(); + bind.sessionSwitchDisplay( + sessionId: gFFI.sessionId, value: Int32List.fromList([i])); gFFI.dialogManager.dismissAll(); }, child: Ink( diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index ce4cbe523..433d3c2d4 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -860,6 +860,8 @@ class FfiModel with ChangeNotifier { // Directly switch to the new display without waiting for the response. switchToNewDisplay(int display, SessionID sessionId, String peerId) { + // VideoHandler creation is upon when video frames are received, so either caching commands(don't know next width/height) or stopping recording when switching displays. + parent.target?.recordingModel.onClose(); // no need to wait for the response pi.currentDisplay = display; updateCurDisplay(sessionId); @@ -868,7 +870,6 @@ class FfiModel with ChangeNotifier { } catch (e) { // } - parent.target?.recordingModel.onSwitchDisplay(); } updateBlockInputState(Map evt, String peerId) { @@ -1850,57 +1851,67 @@ class RecordingModel with ChangeNotifier { int? width = parent.target?.canvasModel.getDisplayWidth(); int? height = parent.target?.canvasModel.getDisplayHeight(); if (sessionId == null || width == null || height == null) return; - final currentDisplay = parent.target?.ffiModel.pi.currentDisplay; - if (currentDisplay != kAllDisplayValue) { - bind.sessionRecordScreen( - sessionId: sessionId, - start: true, - display: currentDisplay!, - width: width, - height: height); - } + final pi = parent.target?.ffiModel.pi; + if (pi == null) return; + final currentDisplay = pi.currentDisplay; + if (currentDisplay == kAllDisplayValue) return; + bind.sessionRecordScreen( + sessionId: sessionId, + start: true, + display: currentDisplay, + width: width, + height: height); } toggle() async { if (isIOS) return; final sessionId = parent.target?.sessionId; if (sessionId == null) return; + final pi = parent.target?.ffiModel.pi; + if (pi == null) return; + final currentDisplay = pi.currentDisplay; + if (currentDisplay == kAllDisplayValue) return; _start = !_start; notifyListeners(); - await bind.sessionRecordStatus(sessionId: sessionId, status: _start); + await _sendStatusMessage(sessionId, pi, _start); if (_start) { - final pi = parent.target?.ffiModel.pi; - if (pi != null) { - sessionRefreshVideo(sessionId, pi); + sessionRefreshVideo(sessionId, pi); + if (versionCmp(pi.version, '1.2.4') >= 0) { + // will not receive SwitchDisplay since 1.2.4 + onSwitchDisplay(); } } else { - final currentDisplay = parent.target?.ffiModel.pi.currentDisplay; - if (currentDisplay != kAllDisplayValue) { - bind.sessionRecordScreen( - sessionId: sessionId, - start: false, - display: currentDisplay!, - width: 0, - height: 0); - } - } - } - - onClose() { - if (isIOS) return; - final sessionId = parent.target?.sessionId; - if (sessionId == null) return; - _start = false; - final currentDisplay = parent.target?.ffiModel.pi.currentDisplay; - if (currentDisplay != kAllDisplayValue) { bind.sessionRecordScreen( sessionId: sessionId, start: false, - display: currentDisplay!, + display: currentDisplay, width: 0, height: 0); } } + + onClose() async { + if (isIOS) return; + final sessionId = parent.target?.sessionId; + if (sessionId == null) return; + if (!_start) return; + _start = false; + final pi = parent.target?.ffiModel.pi; + if (pi == null) return; + final currentDisplay = pi.currentDisplay; + if (currentDisplay == kAllDisplayValue) return; + await _sendStatusMessage(sessionId, pi, false); + bind.sessionRecordScreen( + sessionId: sessionId, + start: false, + display: currentDisplay, + width: 0, + height: 0); + } + + _sendStatusMessage(SessionID sessionId, PeerInfo pi, bool status) async { + await bind.sessionRecordStatus(sessionId: sessionId, status: status); + } } class ElevationModel with ChangeNotifier { diff --git a/libs/scrap/src/common/record.rs b/libs/scrap/src/common/record.rs index 09d5efd3f..3c0ee2a95 100644 --- a/libs/scrap/src/common/record.rs +++ b/libs/scrap/src/common/record.rs @@ -49,8 +49,8 @@ impl RecorderContext { } let file = if self.server { "s" } else { "c" }.to_string() + &self.id.clone() - + &chrono::Local::now().format("_%Y%m%d%H%M%S_").to_string() - + &self.format.to_string() + + &chrono::Local::now().format("_%Y%m%d%H%M%S%3f_").to_string() + + &self.format.to_string().to_lowercase() + if self.format == CodecFormat::VP9 || self.format == CodecFormat::VP8 || self.format == CodecFormat::AV1 @@ -86,6 +86,7 @@ pub enum RecordState { pub struct Recorder { pub inner: Box, ctx: RecorderContext, + pts: Option, } impl Deref for Recorder { @@ -109,11 +110,13 @@ impl Recorder { CodecFormat::VP8 | CodecFormat::VP9 | CodecFormat::AV1 => Recorder { inner: Box::new(WebmRecorder::new(ctx.clone())?), ctx, + pts: None, }, #[cfg(feature = "hwcodec")] _ => Recorder { inner: Box::new(HwRecorder::new(ctx.clone())?), ctx, + pts: None, }, #[cfg(not(feature = "hwcodec"))] _ => bail!("unsupported codec type"), @@ -134,6 +137,7 @@ impl Recorder { _ => bail!("unsupported codec type"), }; self.ctx = ctx; + self.pts = None; self.send_state(RecordState::NewFile(self.ctx.filename.clone())); Ok(()) } @@ -155,7 +159,10 @@ impl Recorder { ..self.ctx.clone() })?; } - vp8s.frames.iter().map(|f| self.write_video(f)).count(); + for f in vp8s.frames.iter() { + self.check_pts(f.pts)?; + self.write_video(f); + } } video_frame::Union::Vp9s(vp9s) => { if self.ctx.format != CodecFormat::VP9 { @@ -164,7 +171,10 @@ impl Recorder { ..self.ctx.clone() })?; } - vp9s.frames.iter().map(|f| self.write_video(f)).count(); + for f in vp9s.frames.iter() { + self.check_pts(f.pts)?; + self.write_video(f); + } } video_frame::Union::Av1s(av1s) => { if self.ctx.format != CodecFormat::AV1 { @@ -173,7 +183,10 @@ impl Recorder { ..self.ctx.clone() })?; } - av1s.frames.iter().map(|f| self.write_video(f)).count(); + for f in av1s.frames.iter() { + self.check_pts(f.pts)?; + self.write_video(f); + } } #[cfg(feature = "hwcodec")] video_frame::Union::H264s(h264s) => { @@ -183,8 +196,9 @@ impl Recorder { ..self.ctx.clone() })?; } - if self.ctx.format == CodecFormat::H264 { - h264s.frames.iter().map(|f| self.write_video(f)).count(); + for f in h264s.frames.iter() { + self.check_pts(f.pts)?; + self.write_video(f); } } #[cfg(feature = "hwcodec")] @@ -195,8 +209,9 @@ impl Recorder { ..self.ctx.clone() })?; } - if self.ctx.format == CodecFormat::H265 { - h265s.frames.iter().map(|f| self.write_video(f)).count(); + for f in h265s.frames.iter() { + self.check_pts(f.pts)?; + self.write_video(f); } } _ => bail!("unsupported frame type"), @@ -205,6 +220,17 @@ impl Recorder { Ok(()) } + fn check_pts(&mut self, pts: i64) -> ResultType<()> { + // https://stackoverflow.com/questions/76379101/how-to-create-one-playable-webm-file-from-two-different-video-tracks-with-same-c + let old_pts = self.pts; + self.pts = Some(pts); + if old_pts.clone().unwrap_or_default() > pts { + log::info!("pts {:?}->{}, change record filename", old_pts, pts); + self.change(self.ctx.clone())?; + } + Ok(()) + } + fn send_state(&self, state: RecordState) { self.ctx.tx.as_ref().map(|tx| tx.send(state)); } @@ -250,7 +276,9 @@ impl RecorderApi for WebmRecorder { if ctx.format == CodecFormat::AV1 { // [129, 8, 12, 0] in 3.6.0, but zero works let codec_private = vec![0, 0, 0, 0]; - webm.set_codec_private(vt.track_number(), &codec_private); + if !webm.set_codec_private(vt.track_number(), &codec_private) { + bail!("Failed to set codec private"); + } } Ok(WebmRecorder { vt, diff --git a/src/client.rs b/src/client.rs index da4559c99..5a3c35467 100644 --- a/src/client.rs +++ b/src/client.rs @@ -999,16 +999,19 @@ pub struct VideoHandler { pub rgb: ImageRgb, recorder: Arc>>, record: bool, + _display: usize, // useful for debug } impl VideoHandler { /// Create a new video handler. - pub fn new() -> Self { + pub fn new(_display: usize) -> Self { + log::info!("new video handler for display #{_display}"); VideoHandler { decoder: Decoder::new(), rgb: ImageRgb::new(ImageFormat::ARGB, crate::DST_STRIDE_RGBA), recorder: Default::default(), record: false, + _display, } } @@ -1900,7 +1903,7 @@ where if handler_controller_map.len() <= display { for _i in handler_controller_map.len()..=display { handler_controller_map.push(VideoHandlerController { - handler: VideoHandler::new(), + handler: VideoHandler::new(_i), count: 0, duration: std::time::Duration::ZERO, skip_beginning: 0, @@ -1960,6 +1963,7 @@ where } } MediaData::RecordScreen(start, display, w, h, id) => { + log::info!("record screen command: start:{start}, display:{display}"); if handler_controller_map.len() == 1 { // Compatible with the sciter version(single ui session). // For the sciter version, there're no multi-ui-sessions for one connection. diff --git a/src/server/connection.rs b/src/server/connection.rs index 1b005ea5e..86096288d 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -2104,7 +2104,7 @@ impl Connection { display, video_service::OPTION_REFRESH, super::service::SERVICE_OPTION_VALUE_TRUE, - ) + ); }); } @@ -2145,6 +2145,7 @@ impl Connection { // Send display changed message. // For compatibility with old versions ( < 1.2.4 ). + // sciter need it in new version if let Some(msg_out) = video_service::make_display_changed_msg(self.display_idx, None) { self.send(msg_out).await; } diff --git a/src/ui/header.tis b/src/ui/header.tis index 838514234..029dd2629 100644 --- a/src/ui/header.tis +++ b/src/ui/header.tis @@ -299,14 +299,22 @@ class Header: Reactor.Component { header.update(); handler.record_status(recording); // 0 is just a dummy value. It will be ignored by the handler. - if (recording) + if (recording) { handler.refresh_video(0); - else - handler.record_screen(false, 0, display_width, display_height); + if (handler.version_cmp(pi.version, '1.2.4') >= 0) handler.record_screen(recording, pi.current_display, display_width, display_height); + } + else { + handler.record_screen(recording, pi.current_display, display_width, display_height); + } } event click $(#screen) (_, me) { if (pi.current_display == me.index) return; + if (recording) { + recording = false; + handler.record_screen(false, pi.current_display, display_width, display_height); + handler.record_status(false); + } handler.switch_display(me.index); } diff --git a/src/ui/remote.rs b/src/ui/remote.rs index 71ef8f84d..bb01ef9f4 100644 --- a/src/ui/remote.rs +++ b/src/ui/remote.rs @@ -243,6 +243,7 @@ impl InvokeUiSession for SciterHandler { pi_sciter.set_item("sas_enabled", pi.sas_enabled); pi_sciter.set_item("displays", Self::make_displays_array(&pi.displays)); pi_sciter.set_item("current_display", pi.current_display); + pi_sciter.set_item("version", pi.version.clone()); self.call("updatePi", &make_args!(pi_sciter)); } @@ -469,6 +470,7 @@ impl sciter::EventHandler for SciterSession { fn restart_remote_device(); fn request_voice_call(); fn close_voice_call(); + fn version_cmp(String, String); } } @@ -757,6 +759,10 @@ impl SciterSession { log::error!("Failed to spawn IP tunneling: {}", err); } } + + fn version_cmp(&self, v1: String, v2: String) -> i32 { + (hbb_common::get_version_number(&v1) - hbb_common::get_version_number(&v2)) as i32 + } } pub fn make_fd(id: i32, entries: &Vec, only_count: bool) -> Value {