From 9905695699e105109bfefc41f7cf5538b05aee90 Mon Sep 17 00:00:00 2001 From: 21pages Date: Mon, 7 Aug 2023 21:32:36 +0800 Subject: [PATCH] enable keyframe interval when recording Signed-off-by: 21pages --- flutter/lib/models/model.dart | 3 +- libs/hbb_common/protos/message.proto | 1 + libs/scrap/examples/benchmark.rs | 3 +- libs/scrap/examples/record-screen.rs | 2 +- libs/scrap/src/common/aom.rs | 8 ++- libs/scrap/src/common/codec.rs | 1 + libs/scrap/src/common/hwcodec.rs | 3 +- libs/scrap/src/common/vpxcodec.rs | 18 ++++--- src/flutter_ffi.rs | 6 +++ src/server/connection.rs | 4 ++ src/server/video_qos.rs | 11 ++++ src/server/video_service.rs | 80 ++++++++++++++++------------ src/ui/header.tis | 1 + src/ui/remote.rs | 1 + src/ui_session_interface.rs | 8 +++ 15 files changed, 105 insertions(+), 45 deletions(-) diff --git a/flutter/lib/models/model.dart b/flutter/lib/models/model.dart index c9bcc0477..9fa73e1b7 100644 --- a/flutter/lib/models/model.dart +++ b/flutter/lib/models/model.dart @@ -1532,12 +1532,13 @@ class RecordingModel with ChangeNotifier { sessionId: sessionId, start: true, width: width, height: height); } - toggle() { + toggle() async { if (isIOS) return; final sessionId = parent.target?.sessionId; if (sessionId == null) return; _start = !_start; notifyListeners(); + await bind.sessionRecordStatus(sessionId: sessionId, status: _start); if (_start) { bind.sessionRefresh(sessionId: sessionId); } else { diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index dac137e1c..e6862bc80 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -664,6 +664,7 @@ message Misc { PluginFailure plugin_failure = 26; uint32 full_speed_fps = 27; uint32 auto_adjust_fps = 28; + bool client_record_status = 29; } } diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index f8f2e9436..83fba99d7 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -114,9 +114,9 @@ fn test_vpx( let config = EncoderCfg::VPX(VpxEncoderConfig { width: width as _, height: height as _, - timebase: [1, 1000], quality, codec: codec_id, + keyframe_interval: None, }); let mut encoder = VpxEncoder::new(config).unwrap(); let mut vpxs = vec![]; @@ -161,6 +161,7 @@ fn test_av1(yuvs: &Vec>, width: usize, height: usize, quality: Q, yuv_co width: width as _, height: height as _, quality, + keyframe_interval: None, }); let mut encoder = AomEncoder::new(config).unwrap(); let start = Instant::now(); diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 2430e2872..6640d5698 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -113,9 +113,9 @@ fn main() -> io::Result<()> { let mut vpx = vpx_encode::VpxEncoder::new(EncoderCfg::VPX(vpx_encode::VpxEncoderConfig { width, height, - timebase: [1, 1000], quality, codec: vpx_codec, + keyframe_interval: None, })) .unwrap(); diff --git a/libs/scrap/src/common/aom.rs b/libs/scrap/src/common/aom.rs index f677858a7..dcb4968e6 100644 --- a/libs/scrap/src/common/aom.rs +++ b/libs/scrap/src/common/aom.rs @@ -45,6 +45,7 @@ pub struct AomEncoderConfig { pub width: u32, pub height: u32, pub quality: Quality, + pub keyframe_interval: Option, } pub struct AomEncoder { @@ -105,7 +106,12 @@ mod webrtc { c.g_timebase.num = 1; c.g_timebase.den = kRtpTicksPerSecond; c.g_input_bit_depth = kBitDepth; - c.kf_mode = aom_kf_mode::AOM_KF_DISABLED; + if let Some(keyframe_interval) = cfg.keyframe_interval { + c.kf_min_dist = 0; + c.kf_max_dist = keyframe_interval as _; + } else { + c.kf_mode = aom_kf_mode::AOM_KF_DISABLED; + } let (q_min, q_max, b) = AomEncoder::convert_quality(cfg.quality); if q_min > 0 && q_min < q_max && q_max < 64 { c.rc_min_quantizer = q_min; diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index fac5f82d7..2c3cbff6c 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -45,6 +45,7 @@ pub struct HwEncoderConfig { pub width: usize, pub height: usize, pub quality: Quality, + pub keyframe_interval: Option, } #[derive(Debug, Clone)] diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index ba834e048..c1fbdfa6e 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -52,6 +52,7 @@ impl EncoderApi for HwEncoder { if base_bitrate <= 0 { bitrate = base_bitrate; } + let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32; let ctx = EncodeContext { name: config.name.clone(), width: config.width as _, @@ -60,7 +61,7 @@ impl EncoderApi for HwEncoder { align: HW_STRIDE_ALIGN as _, bitrate: bitrate as i32 * 1000, timebase: DEFAULT_TIME_BASE, - gop: DEFAULT_GOP, + gop, quality: DEFAULT_HW_QUALITY, rc: DEFAULT_RC, thread_count: codec_thread_num() as _, // ffmpeg's thread_count is used for cpu diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index d17aad676..1bdde0ca6 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -65,8 +65,8 @@ impl EncoderApi for VpxEncoder { c.g_w = config.width; c.g_h = config.height; - c.g_timebase.num = config.timebase[0]; - c.g_timebase.den = config.timebase[1]; + c.g_timebase.num = 1; + c.g_timebase.den = 1000; // Output timestamp precision c.rc_undershoot_pct = 95; // When the data buffer falls below this percentage of fullness, a dropped frame is indicated. Set the threshold to zero (0) to disable this feature. // In dynamic scenes, low bitrate gets low fps while high bitrate gets high fps. @@ -76,9 +76,13 @@ impl EncoderApi for VpxEncoder { // https://developers.google.com/media/vp9/bitrate-modes/ // Constant Bitrate mode (CBR) is recommended for live streaming with VP9. c.rc_end_usage = vpx_rc_mode::VPX_CBR; - // c.kf_min_dist = 0; - // c.kf_max_dist = 999999; - c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot + if let Some(keyframe_interval) = config.keyframe_interval { + c.kf_min_dist = 0; + c.kf_max_dist = keyframe_interval as _; + } else { + c.kf_mode = vpx_kf_mode::VPX_KF_DISABLED; // reduce bandwidth a lot + } + let (q_min, q_max, b) = Self::convert_quality(config.quality); if q_min > 0 && q_min < q_max && q_max < 64 { c.rc_min_quantizer = q_min; @@ -343,12 +347,12 @@ pub struct VpxEncoderConfig { pub width: c_uint, /// The height (in pixels). pub height: c_uint, - /// The timebase numerator and denominator (in seconds). - pub timebase: [c_int; 2], /// The image quality pub quality: Quality, /// The codec pub codec: VpxVideoCodecId, + /// keyframe interval + pub keyframe_interval: Option, } #[derive(Clone, Copy, Debug)] diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index c4e9c2eb3..1287abf34 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -174,6 +174,12 @@ pub fn session_record_screen(session_id: SessionID, start: bool, width: usize, h } } +pub fn session_record_status(session_id: SessionID, status: bool) { + if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { + session.record_status(status); + } +} + pub fn session_reconnect(session_id: SessionID, force_relay: bool) { if let Some(session) = SESSIONS.read().unwrap().get(&session_id) { session.reconnect(force_relay); diff --git a/src/server/connection.rs b/src/server/connection.rs index f3e059696..e32a4c1c3 100644 --- a/src/server/connection.rs +++ b/src/server/connection.rs @@ -1907,6 +1907,10 @@ impl Connection { .lock() .unwrap() .user_auto_adjust_fps(self.inner.id(), fps), + Some(misc::Union::ClientRecordStatus(status)) => video_service::VIDEO_QOS + .lock() + .unwrap() + .user_record(self.inner.id(), status), _ => {} }, Some(message::Union::AudioFrame(frame)) => { diff --git a/src/server/video_qos.rs b/src/server/video_qos.rs index 2ebcdaa84..e9eb9dc79 100644 --- a/src/server/video_qos.rs +++ b/src/server/video_qos.rs @@ -36,6 +36,7 @@ struct UserData { quality: Option<(i64, Quality)>, // (time, quality) delay: Option, response_delayed: bool, + record: bool, } pub struct VideoQoS { @@ -114,6 +115,10 @@ impl VideoQoS { self.quality } + pub fn record(&self) -> bool { + self.users.iter().any(|u| u.1.record) + } + pub fn abr_enabled() -> bool { "N" != Config::get_option("enable-abr") } @@ -388,6 +393,12 @@ impl VideoQoS { } } + pub fn user_record(&mut self, id: i32, v: bool) { + if let Some(user) = self.users.get_mut(&id) { + user.record = v; + } + } + pub fn on_connection_close(&mut self, id: i32) { self.users.remove(&id); self.refresh(None); diff --git a/src/server/video_service.rs b/src/server/video_service.rs index bb0697409..a74211fcd 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -36,7 +36,7 @@ use hbb_common::{ use scrap::Capturer; use scrap::{ aom::AomEncoderConfig, - codec::{Encoder, EncoderCfg, HwEncoderConfig}, + codec::{Encoder, EncoderCfg, HwEncoderConfig, Quality}, record::{Recorder, RecorderContext}, vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, CodecName, Display, TraitCapturer, @@ -518,37 +518,13 @@ fn run(sp: GenericService) -> ResultType<()> { let mut spf; let mut quality = video_qos.quality(); let abr = VideoQoS::abr_enabled(); - drop(video_qos); log::info!("init quality={:?}, abr enabled:{}", quality, abr); - - let encoder_cfg = match Encoder::negotiated_codec() { - scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => { - EncoderCfg::HW(HwEncoderConfig { - name, - width: c.width, - height: c.height, - quality, - }) - } - name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => { - EncoderCfg::VPX(VpxEncoderConfig { - width: c.width as _, - height: c.height as _, - timebase: [1, 1000], // Output timestamp precision - quality, - codec: if name == scrap::CodecName::VP8 { - VpxVideoCodecId::VP8 - } else { - VpxVideoCodecId::VP9 - }, - }) - } - scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig { - width: c.width as _, - height: c.height as _, - quality, - }), - }; + let codec_name = Encoder::negotiated_codec(); + let recorder = get_recorder(c.width, c.height, &codec_name); + let last_recording = + (recorder.lock().unwrap().is_some() || video_qos.record()) && codec_name != CodecName::AV1; + drop(video_qos); + let encoder_cfg = get_encoder_config(&c, quality, last_recording); let mut encoder; match Encoder::new(encoder_cfg) { @@ -597,8 +573,6 @@ fn run(sp: GenericService) -> ResultType<()> { let mut try_gdi = 1; #[cfg(windows)] log::info!("gdi: {}", c.is_gdi()); - let codec_name = Encoder::negotiated_codec(); - let recorder = get_recorder(c.width, c.height, &codec_name); #[cfg(windows)] start_uac_elevation_check(); @@ -617,6 +591,11 @@ fn run(sp: GenericService) -> ResultType<()> { allow_err!(encoder.set_quality(quality)); video_qos.store_bitrate(encoder.bitrate()); } + let recording = (recorder.lock().unwrap().is_some() || video_qos.record()) + && codec_name != CodecName::AV1; + if recording != last_recording { + bail!("SWITCH"); + } drop(video_qos); if *SWITCH.lock().unwrap() { @@ -789,6 +768,41 @@ fn run(sp: GenericService) -> ResultType<()> { Ok(()) } +fn get_encoder_config(c: &CapturerInfo, quality: Quality, recording: bool) -> EncoderCfg { + // https://www.wowza.com/community/t/the-correct-keyframe-interval-in-obs-studio/95162 + let keyframe_interval = if recording { Some(240) } else { None }; + match Encoder::negotiated_codec() { + scrap::CodecName::H264(name) | scrap::CodecName::H265(name) => { + EncoderCfg::HW(HwEncoderConfig { + name, + width: c.width, + height: c.height, + quality, + keyframe_interval, + }) + } + name @ (scrap::CodecName::VP8 | scrap::CodecName::VP9) => { + EncoderCfg::VPX(VpxEncoderConfig { + width: c.width as _, + height: c.height as _, + quality, + codec: if name == scrap::CodecName::VP8 { + VpxVideoCodecId::VP8 + } else { + VpxVideoCodecId::VP9 + }, + keyframe_interval, + }) + } + scrap::CodecName::AV1 => EncoderCfg::AOM(AomEncoderConfig { + width: c.width as _, + height: c.height as _, + quality, + keyframe_interval, + }), + } +} + fn get_recorder( width: usize, height: usize, diff --git a/src/ui/header.tis b/src/ui/header.tis index 666150fb3..2adc37027 100644 --- a/src/ui/header.tis +++ b/src/ui/header.tis @@ -297,6 +297,7 @@ class Header: Reactor.Component { event click $(span#recording) (_, me) { recording = !recording; header.update(); + handler.record_status(recording); if (recording) handler.refresh_video(); else diff --git a/src/ui/remote.rs b/src/ui/remote.rs index 6826c7e54..4ec0d5a5c 100644 --- a/src/ui/remote.rs +++ b/src/ui/remote.rs @@ -451,6 +451,7 @@ impl sciter::EventHandler for SciterSession { fn save_custom_image_quality(i32); fn refresh_video(); fn record_screen(bool, i32, i32); + fn record_status(bool); fn get_toggle_option(String); fn is_privacy_mode_supported(); fn toggle_option(String); diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 704b91cf0..10e8978c9 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -240,6 +240,14 @@ impl Session { self.send(Data::RecordScreen(start, w, h, self.id.clone())); } + pub fn record_status(&self, status: bool) { + let mut misc = Misc::new(); + misc.set_client_record_status(status); + let mut msg = Message::new(); + msg.set_misc(misc); + self.send(Data::Message(msg)); + } + pub fn save_custom_image_quality(&mut self, custom_image_quality: i32) { let msg = self .lc