From ebb14af488f31eb651a4c3c4c873aed7303cf53a Mon Sep 17 00:00:00 2001 From: RustDesk <71636191+rustdesk@users.noreply.github.com> Date: Mon, 11 Dec 2023 23:46:32 +0900 Subject: [PATCH] Revert "use fullrange by default for yuv420p if supported (#6655)" (#6656) This reverts commit 80afa98d667db773d9451aa1ecc9c1d53d2a5828. --- Cargo.lock | 2 +- flutter/lib/common/widgets/toolbar.dart | 5 +- libs/hbb_common/protos/message.proto | 23 +---- libs/scrap/examples/benchmark.rs | 38 ++----- libs/scrap/examples/record-screen.rs | 12 +-- libs/scrap/src/common/aom.rs | 97 +++++------------- libs/scrap/src/common/codec.rs | 129 ++++++------------------ libs/scrap/src/common/convert.rs | 44 +++----- libs/scrap/src/common/hwcodec.rs | 9 +- libs/scrap/src/common/mod.rs | 59 +++-------- libs/scrap/src/common/vpxcodec.rs | 81 +++++---------- src/client/io_loop.rs | 4 +- src/flutter_ffi.rs | 9 -- src/server/video_service.rs | 6 +- src/ui_session_interface.rs | 11 -- 15 files changed, 141 insertions(+), 388 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 298757b60..0dfa4a1ac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3013,7 +3013,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" [[package]] name = "hwcodec" version = "0.1.3" -source = "git+https://github.com/21pages/hwcodec?branch=stable#12fcf7208a2972050e2343a90eaea6e97d21beb0" +source = "git+https://github.com/21pages/hwcodec?branch=stable#83300549075158e5a3fa6c59ea527af3330e48ff" dependencies = [ "bindgen 0.59.2", "cc", diff --git a/flutter/lib/common/widgets/toolbar.dart b/flutter/lib/common/widgets/toolbar.dart index 53b545b33..8bd7cd53e 100644 --- a/flutter/lib/common/widgets/toolbar.dart +++ b/flutter/lib/common/widgets/toolbar.dart @@ -522,9 +522,8 @@ Future> toolbarDisplayToggle( // 444 final codec_format = ffi.qualityMonitorModel.data.codecFormat; - if (codec_format != null && - bind.sessionIsCodecSupport444( - sessionId: sessionId, codec: codec_format)) { + if (versionCmp(pi.version, "1.2.4") >= 0 && + (codec_format == "AV1" || codec_format == "VP9")) { final option = 'i444'; final value = bind.sessionGetToggleOptionSync(sessionId: sessionId, arg: option); diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 0aff3da45..a31655f69 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -18,8 +18,8 @@ message YUV { } enum Chroma { - C420 = 0; - C444 = 1; + I420 = 0; + I444 = 1; } message VideoFrame { @@ -96,28 +96,12 @@ message CodecAbility { bool h265 = 5; } -// vpx, aom have yuv420p_bt601_studio by default -// h26x have nv12_bt601_studio by default -message ColorAbility { - bool yuv420p_bt601_full = 1; - bool yuv444p_bt601_studio = 2; -} - -message ColorAbilities { - ColorAbility vp8 = 1; - ColorAbility vp9 = 2; - ColorAbility av1 = 3; - ColorAbility h264 = 4; - ColorAbility h265 = 5; -} - message SupportedEncoding { bool h264 = 1; bool h265 = 2; bool vp8 = 3; bool av1 = 4; - CodecAbility i444 = 5; // deprecated, use color_abilities instead - ColorAbilities color_abilities = 6; + CodecAbility i444 = 5; } message PeerInfo { @@ -579,7 +563,6 @@ message SupportedDecoding { int32 ability_av1 = 6; CodecAbility i444 = 7; Chroma prefer_chroma = 8; - ColorAbilities color_abilities = 9; } message OptionMessage { diff --git a/libs/scrap/examples/benchmark.rs b/libs/scrap/examples/benchmark.rs index ec8ef99bb..fadf4a052 100644 --- a/libs/scrap/examples/benchmark.rs +++ b/libs/scrap/examples/benchmark.rs @@ -5,9 +5,9 @@ use hbb_common::{ }; use scrap::{ aom::{AomDecoder, AomEncoder, AomEncoderConfig}, - codec::{EncoderApi, EncoderCfg, ExtraEncoderCfg, Quality as Q}, - convert_to_yuv, Capturer, ColorRange, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, - VpxEncoder, VpxEncoderConfig, + codec::{EncoderApi, EncoderCfg, Quality as Q}, + convert_to_yuv, Capturer, Display, TraitCapturer, VpxDecoder, VpxDecoderConfig, VpxEncoder, + VpxEncoderConfig, VpxVideoCodecId::{self, *}, STRIDE_ALIGN, }; @@ -110,15 +110,7 @@ fn test_vpx( codec: codec_id, keyframe_interval: None, }); - let extra = ExtraEncoderCfg { - pixfmt: if i444 { - scrap::Pixfmt::YUV444P - } else { - scrap::Pixfmt::YUV420P - }, - range: ColorRange::Studio, - }; - let mut encoder = VpxEncoder::new(config, extra).unwrap(); + let mut encoder = VpxEncoder::new(config, i444).unwrap(); let mut vpxs = vec![]; let start = Instant::now(); let mut size = 0; @@ -130,7 +122,7 @@ fn test_vpx( match c.frame(std::time::Duration::from_millis(30)) { Ok(frame) => { let tmp_timer = Instant::now(); - convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data).unwrap(); + convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data); for ref frame in encoder .encode(start.elapsed().as_millis() as _, &yuv, STRIDE_ALIGN) .unwrap() @@ -195,15 +187,7 @@ fn test_av1( quality, keyframe_interval: None, }); - let extra = ExtraEncoderCfg { - pixfmt: if i444 { - scrap::Pixfmt::YUV444P - } else { - scrap::Pixfmt::YUV420P - }, - range: ColorRange::Studio, - }; - let mut encoder = AomEncoder::new(config, extra).unwrap(); + let mut encoder = AomEncoder::new(config, i444).unwrap(); let start = Instant::now(); let mut size = 0; let mut av1s: Vec> = vec![]; @@ -215,7 +199,7 @@ fn test_av1( match c.frame(std::time::Duration::from_millis(30)) { Ok(frame) => { let tmp_timer = Instant::now(); - convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data).unwrap(); + convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data); for ref frame in encoder .encode(start.elapsed().as_millis() as _, &yuv, STRIDE_ALIGN) .unwrap() @@ -290,10 +274,6 @@ mod hw { yuv_count: usize, h26xs: &mut Vec>, ) { - let extra = ExtraEncoderCfg { - pixfmt: scrap::Pixfmt::NV12, - range: ColorRange::Studio, - }; let mut encoder = HwEncoder::new( EncoderCfg::HW(HwEncoderConfig { name: info.name.clone(), @@ -302,7 +282,7 @@ mod hw { quality, keyframe_interval: None, }), - extra, + false, ) .unwrap(); let mut size = 0; @@ -315,7 +295,7 @@ mod hw { match c.frame(std::time::Duration::from_millis(30)) { Ok(frame) => { let tmp_timer = Instant::now(); - convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data).unwrap(); + convert_to_yuv(&frame, encoder.yuvfmt(), &mut yuv, &mut mid_data); for ref frame in encoder.encode(&yuv).unwrap() { size += frame.data.len(); diff --git a/libs/scrap/examples/record-screen.rs b/libs/scrap/examples/record-screen.rs index 0fbff5f5c..91ea1ecee 100644 --- a/libs/scrap/examples/record-screen.rs +++ b/libs/scrap/examples/record-screen.rs @@ -13,12 +13,12 @@ use std::time::{Duration, Instant}; use std::{io, thread}; use docopt::Docopt; -use scrap::codec::{EncoderApi, EncoderCfg, ExtraEncoderCfg, Quality as Q}; +use scrap::codec::{EncoderApi, EncoderCfg, Quality as Q}; use webm::mux; use webm::mux::Track; use scrap::{convert_to_yuv, vpxcodec as vpx_encode}; -use scrap::{Capturer, ColorRange, Display, TraitCapturer, STRIDE_ALIGN}; +use scrap::{Capturer, Display, TraitCapturer, STRIDE_ALIGN}; const USAGE: &'static str = " Simple WebM screen capture. @@ -110,10 +110,6 @@ fn main() -> io::Result<()> { Quality::Balanced => Q::Balanced, Quality::Low => Q::Low, }; - let extra = ExtraEncoderCfg { - pixfmt: scrap::Pixfmt::YUV420P, - range: ColorRange::Studio, - }; let mut vpx = vpx_encode::VpxEncoder::new( EncoderCfg::VPX(vpx_encode::VpxEncoderConfig { width, @@ -122,7 +118,7 @@ fn main() -> io::Result<()> { codec: vpx_codec, keyframe_interval: None, }), - extra, + false, ) .unwrap(); @@ -156,7 +152,7 @@ fn main() -> io::Result<()> { if let Ok(frame) = c.frame(Duration::from_millis(0)) { let ms = time.as_secs() * 1000 + time.subsec_millis() as u64; - convert_to_yuv(&frame, vpx.yuvfmt(), &mut yuv, &mut mid_data).unwrap(); + convert_to_yuv(&frame, vpx.yuvfmt(), &mut yuv, &mut mid_data); for frame in vpx.encode(ms as i64, &yuv, STRIDE_ALIGN).unwrap() { vt.add_frame(frame.data, frame.pts as u64 * 1_000_000, frame.key); } diff --git a/libs/scrap/src/common/aom.rs b/libs/scrap/src/common/aom.rs index 5f275b49a..169e8d38e 100644 --- a/libs/scrap/src/common/aom.rs +++ b/libs/scrap/src/common/aom.rs @@ -6,10 +6,10 @@ include!(concat!(env!("OUT_DIR"), "/aom_ffi.rs")); -use crate::codec::{base_bitrate, codec_thread_num, ExtraEncoderCfg, Quality}; +use crate::codec::{base_bitrate, codec_thread_num, Quality}; use crate::{codec::EncoderApi, EncodeFrame, STRIDE_ALIGN}; use crate::{common::GoogleImage, generate_call_macro, generate_call_ptr_macro, Error, Result}; -use crate::{ColorRange, EncodeYuvFormat, Pixfmt}; +use crate::{EncodeYuvFormat, Pixfmt}; use hbb_common::{ anyhow::{anyhow, Context}, bytes::Bytes, @@ -53,7 +53,7 @@ pub struct AomEncoder { ctx: aom_codec_ctx_t, width: usize, height: usize, - extra: ExtraEncoderCfg, + i444: bool, yuvfmt: EncodeYuvFormat, } @@ -98,7 +98,7 @@ mod webrtc { pub fn enc_cfg( i: *const aom_codec_iface, cfg: AomEncoderConfig, - extra: &ExtraEncoderCfg, + i444: bool, ) -> ResultType { let mut c = unsafe { std::mem::MaybeUninit::zeroed().assume_init() }; call_aom!(aom_codec_enc_config_default(i, &mut c, kUsageProfile)); @@ -144,20 +144,12 @@ mod webrtc { c.g_lag_in_frames = kLagInFrames; // No look ahead when lag equals 0. // https://aomedia.googlesource.com/aom/+/refs/tags/v3.6.0/av1/common/enums.h#82 - c.g_profile = if extra.pixfmt == Pixfmt::YUV444P { - 1 - } else { - 0 - }; + c.g_profile = if i444 { 1 } else { 0 }; Ok(c) } - pub fn set_controls( - ctx: *mut aom_codec_ctx_t, - cfg: &aom_codec_enc_cfg, - extra: ExtraEncoderCfg, - ) -> ResultType<()> { + pub fn set_controls(ctx: *mut aom_codec_ctx_t, cfg: &aom_codec_enc_cfg) -> ResultType<()> { use aom_tune_content::*; use aome_enc_control_id::*; macro_rules! call_ctl { @@ -219,45 +211,20 @@ mod webrtc { call_ctl!(ctx, AV1E_SET_ENABLE_SMOOTH_INTERINTRA, 0); call_ctl!(ctx, AV1E_SET_ENABLE_TX64, 0); call_ctl!(ctx, AV1E_SET_MAX_REFERENCE_FRAMES, 3); - // https://github.com/chromium/chromium/blob/327564a4861822c816d35395dfb54d7e5039e6ea/media/video/av1_video_encoder.cc#L662 - call_ctl!( - ctx, - AV1E_SET_COLOR_RANGE, - if extra.range == ColorRange::Full { - aom_color_range::AOM_CR_FULL_RANGE - } else { - aom_color_range::AOM_CR_STUDIO_RANGE - } - ); - call_ctl!( - ctx, - AV1E_SET_COLOR_PRIMARIES, - aom_color_primaries::AOM_CICP_CP_BT_601 - ); - call_ctl!( - ctx, - AV1E_SET_TRANSFER_CHARACTERISTICS, - aom_transfer_characteristics::AOM_CICP_TC_BT_601 - ); - call_ctl!( - ctx, - AV1E_SET_MATRIX_COEFFICIENTS, - aom_matrix_coefficients::AOM_CICP_MC_BT_601 - ); Ok(()) } } impl EncoderApi for AomEncoder { - fn new(cfg: crate::codec::EncoderCfg, extra: ExtraEncoderCfg) -> ResultType + fn new(cfg: crate::codec::EncoderCfg, i444: bool) -> ResultType where Self: Sized, { match cfg { crate::codec::EncoderCfg::AOM(config) => { let i = call_aom_ptr!(aom_codec_av1_cx()); - let c = webrtc::enc_cfg(i, config, &extra)?; + let c = webrtc::enc_cfg(i, config, i444)?; let mut ctx = Default::default(); // Flag options: AOM_CODEC_USE_PSNR and AOM_CODEC_USE_HIGHBITDEPTH @@ -269,13 +236,13 @@ impl EncoderApi for AomEncoder { flags, AOM_ENCODER_ABI_VERSION as _ )); - webrtc::set_controls(&mut ctx, &c, extra)?; + webrtc::set_controls(&mut ctx, &c)?; Ok(Self { ctx, width: config.width as _, height: config.height as _, - extra, - yuvfmt: Self::get_yuvfmt(config.width, config.height, &extra), + i444, + yuvfmt: Self::get_yuvfmt(config.width, config.height, i444), }) } _ => Err(anyhow!("encoder type mismatch")), @@ -324,13 +291,16 @@ impl EncoderApi for AomEncoder { impl AomEncoder { pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { - let (fmt, bpp) = match self.extra.pixfmt { - Pixfmt::YUV444P => (aom_img_fmt::AOM_IMG_FMT_I444, 24), - _ => (aom_img_fmt::AOM_IMG_FMT_I420, 12), - }; + let bpp = if self.i444 { 24 } else { 12 }; if data.len() < self.width * self.height * bpp / 8 { return Err(Error::FailedCall("len not enough".to_string())); } + let fmt = if self.i444 { + aom_img_fmt::AOM_IMG_FMT_I444 + } else { + aom_img_fmt::AOM_IMG_FMT_I420 + }; + let mut image = Default::default(); call_aom_ptr!(aom_img_wrap( &mut image, @@ -405,11 +375,12 @@ impl AomEncoder { (q_min, q_max) } - fn get_yuvfmt(width: u32, height: u32, extra: &ExtraEncoderCfg) -> EncodeYuvFormat { + fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat { let mut img = Default::default(); - let fmt = match extra.pixfmt { - Pixfmt::YUV444P => aom_img_fmt::AOM_IMG_FMT_I444, - _ => aom_img_fmt::AOM_IMG_FMT_I420, + let fmt = if i444 { + aom_img_fmt::AOM_IMG_FMT_I444 + } else { + aom_img_fmt::AOM_IMG_FMT_I420 }; unsafe { aom_img_wrap( @@ -421,9 +392,9 @@ impl AomEncoder { 0x1 as _, ); } + let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 }; EncodeYuvFormat { - pixfmt: extra.pixfmt, - range: extra.range, + pixfmt, w: img.w as _, h: img.h as _, stride: img.stride.map(|s| s as usize).to_vec(), @@ -599,22 +570,8 @@ impl GoogleImage for Image { fn chroma(&self) -> Chroma { match self.inner().fmt { - aom_img_fmt::AOM_IMG_FMT_I444 => Chroma::C444, - _ => Chroma::C420, - } - } - - fn pixfmt(&self) -> Pixfmt { - match self.inner().fmt { - aom_img_fmt::AOM_IMG_FMT_I444 => Pixfmt::YUV444P, - _ => Pixfmt::YUV420P, - } - } - - fn range(&self) -> ColorRange { - match self.inner().range { - aom_color_range::AOM_CR_STUDIO_RANGE => ColorRange::Studio, - aom_color_range::AOM_CR_FULL_RANGE => ColorRange::Full, + aom_img_fmt::AOM_IMG_FMT_I444 => Chroma::I444, + _ => Chroma::I420, } } } diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index dd38eb161..32ebf2dbc 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -14,7 +14,7 @@ use crate::{ aom::{self, AomDecoder, AomEncoder, AomEncoderConfig}, common::GoogleImage, vpxcodec::{self, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, VpxVideoCodecId}, - CodecName, ColorRange, EncodeYuvFormat, ImageRgb, Pixfmt, + CodecName, EncodeYuvFormat, ImageRgb, }; use hbb_common::{ @@ -23,8 +23,8 @@ use hbb_common::{ config::PeerConfig, log, message_proto::{ - supported_decoding::PreferCodec, video_frame, Chroma, CodecAbility, ColorAbilities, - ColorAbility, EncodedVideoFrames, SupportedDecoding, SupportedEncoding, VideoFrame, + supported_decoding::PreferCodec, video_frame, Chroma, CodecAbility, EncodedVideoFrames, + SupportedDecoding, SupportedEncoding, VideoFrame, }, sysinfo::System, tokio::time::Instant, @@ -55,14 +55,8 @@ pub enum EncoderCfg { HW(HwEncoderConfig), } -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub struct ExtraEncoderCfg { - pub pixfmt: Pixfmt, - pub range: ColorRange, -} - pub trait EncoderApi { - fn new(cfg: EncoderCfg, extra: ExtraEncoderCfg) -> ResultType + fn new(cfg: EncoderCfg, i444: bool) -> ResultType where Self: Sized; @@ -113,18 +107,18 @@ pub enum EncodingUpdate { } impl Encoder { - pub fn new(config: EncoderCfg, extra: ExtraEncoderCfg) -> ResultType { - log::info!("new encoder: {config:?}, extra: {extra:?}"); + pub fn new(config: EncoderCfg, i444: bool) -> ResultType { + log::info!("new encoder: {config:?}, i444: {i444}"); match config { EncoderCfg::VPX(_) => Ok(Encoder { - codec: Box::new(VpxEncoder::new(config, extra)?), + codec: Box::new(VpxEncoder::new(config, i444)?), }), EncoderCfg::AOM(_) => Ok(Encoder { - codec: Box::new(AomEncoder::new(config, extra)?), + codec: Box::new(AomEncoder::new(config, i444)?), }), #[cfg(feature = "hwcodec")] - EncoderCfg::HW(_) => match HwEncoder::new(config, extra) { + EncoderCfg::HW(_) => match HwEncoder::new(config, i444) { Ok(hw) => Ok(Encoder { codec: Box::new(hw), }), @@ -244,22 +238,6 @@ impl Encoder { ..Default::default() }) .into(), - color_abilities: Some(ColorAbilities { - vp9: Some(ColorAbility { - yuv420p_bt601_full: true, - yuv444p_bt601_studio: true, - ..Default::default() - }) - .into(), - av1: Some(ColorAbility { - yuv420p_bt601_full: true, - yuv444p_bt601_studio: true, - ..Default::default() - }) - .into(), - ..Default::default() - }) - .into(), ..Default::default() }; #[cfg(feature = "hwcodec")] @@ -271,54 +249,20 @@ impl Encoder { encoding } - pub fn extra(config: &EncoderCfg) -> ExtraEncoderCfg { + pub fn use_i444(config: &EncoderCfg) -> bool { let decodings = PEER_DECODINGS.lock().unwrap().clone(); let prefer_i444 = decodings .iter() - .any(|d| d.1.prefer_chroma == Chroma::C444.into()); - let (pixfmt, range) = match config { + .all(|d| d.1.prefer_chroma == Chroma::I444.into()); + let i444_useable = match config { EncoderCfg::VPX(vpx) => match vpx.codec { - VpxVideoCodecId::VP8 => (Pixfmt::YUV420P, ColorRange::Studio), - VpxVideoCodecId::VP9 => { - if prefer_i444 - && decodings.len() > 0 - && decodings - .iter() - .all(|d| d.1.color_abilities.vp9.yuv444p_bt601_studio) - { - (Pixfmt::YUV444P, ColorRange::Studio) - } else if decodings.len() > 0 - && decodings - .iter() - .all(|d| d.1.color_abilities.vp9.yuv420p_bt601_full) - { - (Pixfmt::YUV420P, ColorRange::Full) - } else { - (Pixfmt::YUV420P, ColorRange::Studio) - } - } + VpxVideoCodecId::VP8 => false, + VpxVideoCodecId::VP9 => decodings.iter().all(|d| d.1.i444.vp9), }, - EncoderCfg::AOM(_) => { - if prefer_i444 - && decodings.len() > 0 - && decodings - .iter() - .all(|d| d.1.color_abilities.av1.yuv444p_bt601_studio) - { - (Pixfmt::YUV444P, ColorRange::Studio) - } else if decodings.len() > 0 - && decodings - .iter() - .all(|d| d.1.color_abilities.av1.yuv420p_bt601_full) - { - (Pixfmt::YUV420P, ColorRange::Full) - } else { - (Pixfmt::YUV420P, ColorRange::Studio) - } - } - EncoderCfg::HW(_) => (Pixfmt::NV12, ColorRange::Studio), + EncoderCfg::AOM(_) => decodings.iter().all(|d| d.1.i444.av1), + EncoderCfg::HW(_) => false, }; - ExtraEncoderCfg { pixfmt, range } + prefer_i444 && i444_useable && !decodings.is_empty() } } @@ -331,7 +275,12 @@ impl Decoder { ability_vp8: 1, ability_vp9: 1, ability_av1: 1, - color_abilities: Some(Self::color_abilities()).into(), + i444: Some(CodecAbility { + vp9: true, + av1: true, + ..Default::default() + }) + .into(), prefer: prefer.into(), prefer_chroma: prefer_chroma.into(), ..Default::default() @@ -422,7 +371,7 @@ impl Decoder { } #[cfg(feature = "hwcodec")] video_frame::Union::H264s(h264s) => { - *chroma = Some(Chroma::C420); + *chroma = Some(Chroma::I420); if let Some(decoder) = &mut self.hw.h264 { Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420) } else { @@ -431,7 +380,7 @@ impl Decoder { } #[cfg(feature = "hwcodec")] video_frame::Union::H265s(h265s) => { - *chroma = Some(Chroma::C420); + *chroma = Some(Chroma::I420); if let Some(decoder) = &mut self.hw.h265 { Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420) } else { @@ -440,7 +389,7 @@ impl Decoder { } #[cfg(feature = "mediacodec")] video_frame::Union::H264s(h264s) => { - *chroma = Some(Chroma::C420); + *chroma = Some(Chroma::I420); if let Some(decoder) = &mut self.media_codec.h264 { Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb) } else { @@ -449,7 +398,7 @@ impl Decoder { } #[cfg(feature = "mediacodec")] video_frame::Union::H265s(h265s) => { - *chroma = Some(Chroma::C420); + *chroma = Some(Chroma::I420); if let Some(decoder) = &mut self.media_codec.h265 { Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb) } else { @@ -551,7 +500,7 @@ impl Decoder { fn preference(id: Option<&str>) -> (PreferCodec, Chroma) { let id = id.unwrap_or_default(); if id.is_empty() { - return (PreferCodec::Auto, Chroma::C420); + return (PreferCodec::Auto, Chroma::I420); } let options = PeerConfig::load(id).options; let codec = options @@ -571,30 +520,12 @@ impl Decoder { PreferCodec::Auto }; let chroma = if options.get("i444") == Some(&"Y".to_string()) { - Chroma::C444 + Chroma::I444 } else { - Chroma::C420 + Chroma::I420 }; (codec, chroma) } - - fn color_abilities() -> ColorAbilities { - ColorAbilities { - vp9: Some(ColorAbility { - yuv420p_bt601_full: true, - yuv444p_bt601_studio: true, - ..Default::default() - }) - .into(), - av1: Some(ColorAbility { - yuv420p_bt601_full: true, - yuv444p_bt601_studio: true, - ..Default::default() - }) - .into(), - ..Default::default() - } - } } #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] diff --git a/libs/scrap/src/common/convert.rs b/libs/scrap/src/common/convert.rs index 42da77e1a..3bf0e7a03 100644 --- a/libs/scrap/src/common/convert.rs +++ b/libs/scrap/src/common/convert.rs @@ -8,7 +8,7 @@ include!(concat!(env!("OUT_DIR"), "/yuv_ffi.rs")); #[cfg(not(target_os = "ios"))] use crate::Frame; -use crate::{generate_call_macro, ColorRange, EncodeYuvFormat, Pixfmt, TraitFrame}; +use crate::{generate_call_macro, EncodeYuvFormat, TraitFrame}; use hbb_common::{bail, log, ResultType}; generate_call_macro!(call_yuv, false); @@ -212,7 +212,7 @@ pub fn convert_to_yuv( dst_fmt.h ); } - if src_pixfmt == Pixfmt::BGRA || src_pixfmt == Pixfmt::RGBA { + if src_pixfmt == crate::Pixfmt::BGRA || src_pixfmt == crate::Pixfmt::RGBA { if src.len() < src_stride[0] * src_height { bail!( "wrong src len, {} < {} * {}", @@ -222,35 +222,22 @@ pub fn convert_to_yuv( ); } } - let align = |x: usize| (x + 63) / 64 * 64; + let align = |x:usize| { + (x + 63) / 64 * 64 + }; match (src_pixfmt, dst_fmt.pixfmt) { - (Pixfmt::BGRA, Pixfmt::YUV420P) | (Pixfmt::RGBA, Pixfmt::YUV420P) => { + (crate::Pixfmt::BGRA, crate::Pixfmt::I420) | (crate::Pixfmt::RGBA, crate::Pixfmt::I420) => { let dst_stride_y = dst_fmt.stride[0]; let dst_stride_uv = dst_fmt.stride[1]; dst.resize(dst_fmt.h * dst_stride_y * 2, 0); // waste some memory to ensure memory safety let dst_y = dst.as_mut_ptr(); let dst_u = dst[dst_fmt.u..].as_mut_ptr(); let dst_v = dst[dst_fmt.v..].as_mut_ptr(); - let mut src = src; - let f = match (src_pixfmt, dst_fmt.pixfmt, dst_fmt.range) { - (Pixfmt::BGRA, Pixfmt::YUV420P, ColorRange::Studio) => ARGBToI420, - (Pixfmt::RGBA, Pixfmt::YUV420P, ColorRange::Studio) => ABGRToI420, - (Pixfmt::BGRA, Pixfmt::YUV420P, ColorRange::Full) => ARGBToJ420, - (Pixfmt::RGBA, Pixfmt::YUV420P, ColorRange::Full) => { - mid_data.resize(src.len(), 0); - call_yuv!(ABGRToARGB( - src.as_ptr(), - src_stride[0] as _, - mid_data.as_mut_ptr(), - src_stride[0] as _, - src_width as _, - src_height as _, - )); - src = mid_data; - ARGBToJ420 - } - _ => bail!("unreachable"), + let f = if src_pixfmt == crate::Pixfmt::BGRA { + ARGBToI420 + } else { + ABGRToI420 }; call_yuv!(f( src.as_ptr(), @@ -265,7 +252,7 @@ pub fn convert_to_yuv( src_height as _, )); } - (Pixfmt::BGRA, Pixfmt::NV12) | (Pixfmt::RGBA, Pixfmt::NV12) => { + (crate::Pixfmt::BGRA, crate::Pixfmt::NV12) | (crate::Pixfmt::RGBA, crate::Pixfmt::NV12) => { let dst_stride_y = dst_fmt.stride[0]; let dst_stride_uv = dst_fmt.stride[1]; dst.resize( @@ -274,7 +261,7 @@ pub fn convert_to_yuv( ); let dst_y = dst.as_mut_ptr(); let dst_uv = dst[dst_fmt.u..].as_mut_ptr(); - let f = if src_pixfmt == Pixfmt::BGRA { + let f = if src_pixfmt == crate::Pixfmt::BGRA { ARGBToNV12 } else { ABGRToNV12 @@ -290,19 +277,18 @@ pub fn convert_to_yuv( src_height as _, )); } - (Pixfmt::BGRA, Pixfmt::YUV444P) | (Pixfmt::RGBA, Pixfmt::YUV444P) => { + (crate::Pixfmt::BGRA, crate::Pixfmt::I444) | (crate::Pixfmt::RGBA, crate::Pixfmt::I444) => { let dst_stride_y = dst_fmt.stride[0]; let dst_stride_u = dst_fmt.stride[1]; let dst_stride_v = dst_fmt.stride[2]; dst.resize( - align(dst_fmt.h) - * (align(dst_stride_y) + align(dst_stride_u) + align(dst_stride_v)), + align(dst_fmt.h) * (align(dst_stride_y) + align(dst_stride_u) + align(dst_stride_v)), 0, ); let dst_y = dst.as_mut_ptr(); let dst_u = dst[dst_fmt.u..].as_mut_ptr(); let dst_v = dst[dst_fmt.v..].as_mut_ptr(); - let src = if src_pixfmt == Pixfmt::BGRA { + let src = if src_pixfmt == crate::Pixfmt::BGRA { src } else { mid_data.resize(src.len(), 0); diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index c806323e9..2aeceb37e 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -1,5 +1,5 @@ use crate::{ - codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg, ExtraEncoderCfg}, + codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg}, hw, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN, }; use hbb_common::{ @@ -36,11 +36,10 @@ pub struct HwEncoder { width: u32, height: u32, bitrate: u32, //kbs - extra: ExtraEncoderCfg, } impl EncoderApi for HwEncoder { - fn new(cfg: EncoderCfg, extra: ExtraEncoderCfg) -> ResultType + fn new(cfg: EncoderCfg, _i444: bool) -> ResultType where Self: Sized, { @@ -83,7 +82,6 @@ impl EncoderApi for HwEncoder { width: ctx.width as _, height: ctx.height as _, bitrate, - extra, }), Err(_) => Err(anyhow!(format!("Failed to create encoder"))), } @@ -122,7 +120,7 @@ impl EncoderApi for HwEncoder { let pixfmt = if self.pixfmt == AVPixelFormat::AV_PIX_FMT_NV12 { Pixfmt::NV12 } else { - Pixfmt::YUV420P + Pixfmt::I420 }; let stride = self .encoder @@ -133,7 +131,6 @@ impl EncoderApi for HwEncoder { .collect(); crate::EncodeYuvFormat { pixfmt, - range: self.extra.range, w: self.encoder.ctx.width as _, h: self.encoder.ctx.height as _, stride, diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 33d910b54..4df0f0192 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -50,7 +50,7 @@ pub mod record; mod vpx; #[repr(usize)] -#[derive(Debug, Copy, Clone)] +#[derive(Copy, Clone)] pub enum ImageFormat { Raw, ABGR, @@ -124,21 +124,14 @@ pub trait TraitFrame { pub enum Pixfmt { BGRA, RGBA, - YUV420P, + I420, NV12, - YUV444P, -} - -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub enum ColorRange { - Studio, - Full, + I444, } #[derive(Debug, Clone)] pub struct EncodeYuvFormat { pub pixfmt: Pixfmt, - pub range: ColorRange, pub w: usize, pub h: usize, pub stride: Vec, @@ -299,8 +292,6 @@ pub trait GoogleImage { fn stride(&self) -> Vec; fn planes(&self) -> Vec<*mut u8>; fn chroma(&self) -> Chroma; - fn pixfmt(&self) -> Pixfmt; - fn range(&self) -> ColorRange; fn get_bytes_per_row(w: usize, fmt: ImageFormat, stride: usize) -> usize { let bytes_per_pixel = match fmt { ImageFormat::Raw => 3, @@ -319,13 +310,9 @@ pub trait GoogleImage { let stride = self.stride(); let planes = self.planes(); unsafe { - match (self.pixfmt(), rgb.fmt()) { - (Pixfmt::YUV420P, ImageFormat::Raw) => { - let f = match self.range() { - ColorRange::Studio => super::I420ToRAW, - ColorRange::Full => super::J420ToRAW, - }; - f( + match (self.chroma(), rgb.fmt()) { + (Chroma::I420, ImageFormat::Raw) => { + super::I420ToRAW( planes[0], stride[0], planes[1], @@ -338,12 +325,8 @@ pub trait GoogleImage { self.height() as _, ); } - (Pixfmt::YUV420P, ImageFormat::ARGB) => { - let f = match self.range() { - ColorRange::Studio => super::I420ToARGB, - ColorRange::Full => super::J420ToARGB, - }; - f( + (Chroma::I420, ImageFormat::ARGB) => { + super::I420ToARGB( planes[0], stride[0], planes[1], @@ -356,12 +339,8 @@ pub trait GoogleImage { self.height() as _, ); } - (Pixfmt::YUV420P, ImageFormat::ABGR) => { - let f = match self.range() { - ColorRange::Studio => super::I420ToABGR, - ColorRange::Full => super::J420ToABGR, - }; - f( + (Chroma::I420, ImageFormat::ABGR) => { + super::I420ToABGR( planes[0], stride[0], planes[1], @@ -374,12 +353,8 @@ pub trait GoogleImage { self.height() as _, ); } - (Pixfmt::YUV444P, ImageFormat::ARGB) => { - let f = match self.range() { - ColorRange::Studio => super::I444ToARGB, - ColorRange::Full => super::J444ToARGB, - }; - f( + (Chroma::I444, ImageFormat::ARGB) => { + super::I444ToARGB( planes[0], stride[0], planes[1], @@ -392,12 +367,8 @@ pub trait GoogleImage { self.height() as _, ); } - (Pixfmt::YUV444P, ImageFormat::ABGR) => { - let f = match self.range() { - ColorRange::Studio => super::I444ToABGR, - ColorRange::Full => super::J444ToABGR, - }; - f( + (Chroma::I444, ImageFormat::ABGR) => { + super::I444ToABGR( planes[0], stride[0], planes[1], @@ -411,7 +382,7 @@ pub trait GoogleImage { ); } // (Chroma::I444, ImageFormat::Raw), new version libyuv have I444ToRAW - _ => log::error!("{:?} -> {:?} is not supported", self.pixfmt(), rgb.fmt()), + _ => log::error!("unsupported pixfmt: {:?}", self.chroma()), } } } diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index 5e3c96715..63154bb1f 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -7,8 +7,8 @@ use hbb_common::log; use hbb_common::message_proto::{Chroma, EncodedVideoFrame, EncodedVideoFrames, VideoFrame}; use hbb_common::ResultType; -use crate::codec::{base_bitrate, codec_thread_num, EncoderApi, ExtraEncoderCfg, Quality}; -use crate::{ColorRange, EncodeYuvFormat, GoogleImage, Pixfmt, STRIDE_ALIGN}; +use crate::codec::{base_bitrate, codec_thread_num, EncoderApi, Quality}; +use crate::{EncodeYuvFormat, GoogleImage, Pixfmt, STRIDE_ALIGN}; use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; use crate::{generate_call_macro, generate_call_ptr_macro, Error, Result}; @@ -39,7 +39,7 @@ pub struct VpxEncoder { width: usize, height: usize, id: VpxVideoCodecId, - extra: ExtraEncoderCfg, + i444: bool, yuvfmt: EncodeYuvFormat, } @@ -48,7 +48,7 @@ pub struct VpxDecoder { } impl EncoderApi for VpxEncoder { - fn new(cfg: crate::codec::EncoderCfg, extra: ExtraEncoderCfg) -> ResultType + fn new(cfg: crate::codec::EncoderCfg, i444: bool) -> ResultType where Self: Sized, { @@ -102,12 +102,11 @@ impl EncoderApi for VpxEncoder { } // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp9/common/vp9_enums.h#29 // https://chromium.googlesource.com/webm/libvpx/+/refs/heads/main/vp8/vp8_cx_iface.c#282 - c.g_profile = - if extra.pixfmt == Pixfmt::YUV444P && config.codec == VpxVideoCodecId::VP9 { - 1 - } else { - 0 - }; + c.g_profile = if i444 && config.codec == VpxVideoCodecId::VP9 { + 1 + } else { + 0 + }; /* The VPX encoder supports two-pass encoding for rate control purposes. @@ -165,22 +164,6 @@ impl EncoderApi for VpxEncoder { VP9E_SET_TILE_COLUMNS as _, 4 as c_int )); - - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_COLOR_RANGE as _, - if extra.range == ColorRange::Full { - vpx_color_range::VPX_CR_FULL_RANGE - } else { - vpx_color_range::VPX_CR_STUDIO_RANGE - } - )); - - call_vpx!(vpx_codec_control_( - &mut ctx, - VP9E_SET_COLOR_SPACE as _, - vpx_color_space::VPX_CS_BT_601 - )); } else if config.codec == VpxVideoCodecId::VP8 { // https://github.com/webmproject/libvpx/blob/972149cafeb71d6f08df89e91a0130d6a38c4b15/vpx/vp8cx.h#L172 // https://groups.google.com/a/webmproject.org/g/webm-discuss/c/DJhSrmfQ61M @@ -192,8 +175,8 @@ impl EncoderApi for VpxEncoder { width: config.width as _, height: config.height as _, id: config.codec, - extra, - yuvfmt: Self::get_yuvfmt(config.width, config.height, &extra), + i444, + yuvfmt: Self::get_yuvfmt(config.width, config.height, i444), }) } _ => Err(anyhow!("encoder type mismatch")), @@ -247,13 +230,16 @@ impl EncoderApi for VpxEncoder { impl VpxEncoder { pub fn encode(&mut self, pts: i64, data: &[u8], stride_align: usize) -> Result { - let (fmt, bpp) = match self.extra.pixfmt { - Pixfmt::YUV444P => (vpx_img_fmt::VPX_IMG_FMT_I444, 24), - _ => (vpx_img_fmt::VPX_IMG_FMT_I420, 12), - }; + let bpp = if self.i444 { 24 } else { 12 }; if data.len() < self.width * self.height * bpp / 8 { return Err(Error::FailedCall("len not enough".to_string())); } + let fmt = if self.i444 { + vpx_img_fmt::VPX_IMG_FMT_I444 + } else { + vpx_img_fmt::VPX_IMG_FMT_I420 + }; + let mut image = Default::default(); call_vpx_ptr!(vpx_img_wrap( &mut image, @@ -351,11 +337,12 @@ impl VpxEncoder { (q_min, q_max) } - fn get_yuvfmt(width: u32, height: u32, extra: &ExtraEncoderCfg) -> EncodeYuvFormat { + fn get_yuvfmt(width: u32, height: u32, i444: bool) -> EncodeYuvFormat { let mut img = Default::default(); - let fmt = match extra.pixfmt { - Pixfmt::YUV444P => vpx_img_fmt::VPX_IMG_FMT_I444, - _ => vpx_img_fmt::VPX_IMG_FMT_I420, + let fmt = if i444 { + vpx_img_fmt::VPX_IMG_FMT_I444 + } else { + vpx_img_fmt::VPX_IMG_FMT_I420 }; unsafe { vpx_img_wrap( @@ -367,9 +354,9 @@ impl VpxEncoder { 0x1 as _, ); } + let pixfmt = if i444 { Pixfmt::I444 } else { Pixfmt::I420 }; EncodeYuvFormat { - pixfmt: extra.pixfmt, - range: extra.range, + pixfmt, w: img.w as _, h: img.h as _, stride: img.stride.map(|s| s as usize).to_vec(), @@ -594,22 +581,8 @@ impl GoogleImage for Image { fn chroma(&self) -> Chroma { match self.inner().fmt { - vpx_img_fmt::VPX_IMG_FMT_I444 => Chroma::C444, - _ => Chroma::C420, - } - } - - fn pixfmt(&self) -> Pixfmt { - match self.inner().fmt { - vpx_img_fmt::VPX_IMG_FMT_I444 => Pixfmt::YUV444P, - _ => Pixfmt::YUV420P, - } - } - - fn range(&self) -> ColorRange { - match self.inner().range { - vpx_color_range::VPX_CR_STUDIO_RANGE => ColorRange::Studio, - vpx_color_range::VPX_CR_FULL_RANGE => ColorRange::Full, + vpx_img_fmt::VPX_IMG_FMT_I444 => Chroma::I444, + _ => Chroma::I420, } } } diff --git a/src/client/io_loop.rs b/src/client/io_loop.rs index f9b4e4a51..ac98d8da8 100644 --- a/src/client/io_loop.rs +++ b/src/client/io_loop.rs @@ -252,8 +252,8 @@ impl Remote { }).collect::>(); let chroma = self.chroma.read().unwrap().clone(); let chroma = match chroma { - Some(Chroma::C444) => "4:4:4", - Some(Chroma::C420) => "4:2:0", + Some(Chroma::I444) => "4:4:4", + Some(Chroma::I420) => "4:2:0", None => "-", }; let chroma = Some(chroma.to_string()); diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index d2733696d..f5791f085 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -1416,15 +1416,6 @@ pub fn session_change_prefer_codec(session_id: SessionID) { } } -pub fn session_is_codec_support_444(session_id: SessionID, codec: String) -> SyncReturn { - let res = if let Some(session) = sessions::get_session_by_session_id(&session_id) { - session.is_codec_support_444(codec) - } else { - false - }; - SyncReturn(res) -} - pub fn session_on_waiting_for_image_dialog_show(session_id: SessionID) { super::flutter::session_on_waiting_for_image_dialog_show(session_id); } diff --git a/src/server/video_service.rs b/src/server/video_service.rs index c18ec9445..cb1eea3ee 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -400,8 +400,8 @@ fn run(vs: VideoService) -> ResultType<()> { let encoder_cfg = get_encoder_config(&c, quality, last_recording); let mut encoder; - let extra = Encoder::extra(&encoder_cfg); - match Encoder::new(encoder_cfg.clone(), extra) { + let use_i444 = Encoder::use_i444(&encoder_cfg); + match Encoder::new(encoder_cfg.clone(), use_i444) { Ok(x) => encoder = x, Err(err) => bail!("Failed to create encoder: {}", err), } @@ -456,7 +456,7 @@ fn run(vs: VideoService) -> ResultType<()> { if last_portable_service_running != crate::portable_service::client::running() { bail!("SWITCH"); } - if Encoder::extra(&encoder_cfg) != extra { + if Encoder::use_i444(&encoder_cfg) != use_i444 { bail!("SWITCH"); } check_privacy_mode_changed(&sp, c.privacy_mode_id)?; diff --git a/src/ui_session_interface.rs b/src/ui_session_interface.rs index 18b754d6e..4c5785640 100644 --- a/src/ui_session_interface.rs +++ b/src/ui_session_interface.rs @@ -1230,17 +1230,6 @@ impl Session { pub fn close_voice_call(&self) { self.send(Data::CloseVoiceCall); } - - pub fn is_codec_support_444(&self, codec: String) -> bool { - let codec = codec.to_uppercase(); - let encoding = self.lc.read().unwrap().supported_encoding.clone(); - // decoding support following - match codec.as_str() { - "VP9" => encoding.color_abilities.vp9.yuv444p_bt601_studio, - "AV1" => encoding.color_abilities.av1.yuv444p_bt601_studio, - _ => false, - } - } } pub trait InvokeUiSession: Send + Sync + Clone + 'static + Sized + Default {