Merge pull request #1013 from 21pages/hwcodec

Hwcodec: codec preference
This commit is contained in:
RustDesk 2022-07-22 01:24:41 +08:00 committed by GitHub
commit eda5a94e33
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
14 changed files with 291 additions and 150 deletions

2
Cargo.lock generated
View File

@ -2212,7 +2212,7 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]]
name = "hwcodec"
version = "0.1.0"
source = "git+https://github.com/21pages/hwcodec#91d1cd327c88490f917457072aeef0676ddb2be7"
source = "git+https://github.com/21pages/hwcodec#890204e0703a3d361fc7a45f035fe75c0575bb1d"
dependencies = [
"bindgen",
"cc",

View File

@ -72,6 +72,11 @@ message Features {
bool privacy_mode = 1;
}
message SupportedEncoding {
bool h264 = 1;
bool h265 = 2;
}
message PeerInfo {
string username = 1;
string hostname = 2;
@ -82,6 +87,7 @@ message PeerInfo {
string version = 7;
int32 conn_id = 8;
Features features = 9;
SupportedEncoding encoding = 10;
}
message LoginResponse {
@ -434,9 +440,17 @@ enum ImageQuality {
}
message VideoCodecState {
int32 ScoreVpx = 1;
int32 ScoreH264 = 2;
int32 ScoreH265 = 3;
enum PerferCodec {
Auto = 0;
VPX = 1;
H264 = 2;
H265 = 3;
}
int32 score_vpx = 1;
int32 score_h264 = 2;
int32 score_h265 = 3;
PerferCodec perfer = 4;
}
message OptionMessage {

View File

@ -978,6 +978,10 @@ impl HwCodecConfig {
pub fn store(&self) {
Config::store_(self, "_hwcodec");
}
pub fn remove() {
std::fs::remove_file(Config::file_("_hwcodec")).ok();
}
}
#[cfg(test)]

View File

@ -50,5 +50,5 @@ gstreamer = { version = "0.16", optional = true }
gstreamer-app = { version = "0.16", features = ["v1_10"], optional = true }
gstreamer-video = { version = "0.16", optional = true }
[target.'cfg(target_os = "windows")'.dependencies]
[target.'cfg(any(target_os = "windows", target_os = "linux"))'.dependencies]
hwcodec = { git = "https://github.com/21pages/hwcodec", optional = true }

View File

@ -16,12 +16,15 @@ use hbb_common::{
ResultType,
};
#[cfg(feature = "hwcodec")]
use hbb_common::{config::Config2, lazy_static};
use hbb_common::{
config::{Config2, PeerConfig},
lazy_static,
message_proto::video_codec_state::PerferCodec,
};
#[cfg(feature = "hwcodec")]
lazy_static::lazy_static! {
static ref PEER_DECODER_STATES: Arc<Mutex<HashMap<i32, VideoCodecState>>> = Default::default();
static ref MY_DECODER_STATE: Arc<Mutex<VideoCodecState>> = Default::default();
}
const SCORE_VPX: i32 = 90;
@ -102,7 +105,7 @@ impl Encoder {
codec: Box::new(hw),
}),
Err(e) => {
HwEncoder::best(true, true);
check_config_process(true);
Err(e)
}
},
@ -113,7 +116,6 @@ impl Encoder {
// TODO
pub fn update_video_encoder(id: i32, update: EncoderUpdate) {
log::info!("encoder update: {:?}", update);
#[cfg(feature = "hwcodec")]
{
let mut states = PEER_DECODER_STATES.lock().unwrap();
@ -130,49 +132,75 @@ impl Encoder {
}
}
}
let current_encoder_name = HwEncoder::current_name();
let name = HwEncoder::current_name();
if states.len() > 0 {
let (best, _) = HwEncoder::best(false, true);
let best = HwEncoder::best();
let enabled_h264 = best.h264.is_some()
&& states.len() > 0
&& states.iter().all(|(_, s)| s.ScoreH264 > 0);
&& states.iter().all(|(_, s)| s.score_h264 > 0);
let enabled_h265 = best.h265.is_some()
&& states.len() > 0
&& states.iter().all(|(_, s)| s.ScoreH265 > 0);
&& states.iter().all(|(_, s)| s.score_h265 > 0);
// score encoder
let mut score_vpx = SCORE_VPX;
let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score);
let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score);
// score decoder
score_vpx += states.iter().map(|s| s.1.ScoreVpx).sum::<i32>();
if enabled_h264 {
score_h264 += states.iter().map(|s| s.1.ScoreH264).sum::<i32>();
}
if enabled_h265 {
score_h265 += states.iter().map(|s| s.1.ScoreH265).sum::<i32>();
// Preference first
let mut preference = PerferCodec::Auto;
let preferences: Vec<_> = states
.iter()
.filter(|(_, s)| {
s.perfer == PerferCodec::VPX.into()
|| s.perfer == PerferCodec::H264.into() && enabled_h264
|| s.perfer == PerferCodec::H265.into() && enabled_h265
})
.map(|(_, s)| s.perfer)
.collect();
if preferences.len() > 0 && preferences.iter().all(|&p| p == preferences[0]) {
preference = preferences[0].enum_value_or(PerferCodec::Auto);
}
if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 {
*current_encoder_name.lock().unwrap() = Some(best.h265.unwrap().name);
} else if enabled_h264 && score_h264 >= score_vpx && score_h264 >= score_h265 {
*current_encoder_name.lock().unwrap() = Some(best.h264.unwrap().name);
} else {
*current_encoder_name.lock().unwrap() = None;
match preference {
PerferCodec::VPX => *name.lock().unwrap() = None,
PerferCodec::H264 => {
*name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name))
}
PerferCodec::H265 => {
*name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name))
}
PerferCodec::Auto => {
// score encoder
let mut score_vpx = SCORE_VPX;
let mut score_h264 = best.h264.as_ref().map_or(0, |c| c.score);
let mut score_h265 = best.h265.as_ref().map_or(0, |c| c.score);
// score decoder
score_vpx += states.iter().map(|s| s.1.score_vpx).sum::<i32>();
if enabled_h264 {
score_h264 += states.iter().map(|s| s.1.score_h264).sum::<i32>();
}
if enabled_h265 {
score_h265 += states.iter().map(|s| s.1.score_h265).sum::<i32>();
}
if enabled_h265 && score_h265 >= score_vpx && score_h265 >= score_h264 {
*name.lock().unwrap() = best.h265.map_or(None, |c| Some(c.name));
} else if enabled_h264
&& score_h264 >= score_vpx
&& score_h264 >= score_h265
{
*name.lock().unwrap() = best.h264.map_or(None, |c| Some(c.name));
} else {
*name.lock().unwrap() = None;
}
}
}
log::info!(
"connection count:{}, h264:{}, h265:{}, score: vpx({}), h264({}), h265({}), set current encoder name {:?}",
"connection count:{}, used preference:{:?}, encoder:{:?}",
states.len(),
enabled_h264,
enabled_h265,
score_vpx,
score_h264,
score_h265,
current_encoder_name.lock().unwrap()
)
preference,
name.lock().unwrap()
)
} else {
*current_encoder_name.lock().unwrap() = None;
*name.lock().unwrap() = None;
}
}
#[cfg(not(feature = "hwcodec"))]
@ -192,57 +220,57 @@ impl Encoder {
#[cfg(not(feature = "hwcodec"))]
return None;
}
}
#[cfg(feature = "hwcodec")]
impl Drop for Decoder {
fn drop(&mut self) {
*MY_DECODER_STATE.lock().unwrap() = VideoCodecState {
ScoreVpx: SCORE_VPX,
..Default::default()
};
pub fn supported_encoding() -> (bool, bool) {
#[cfg(feature = "hwcodec")]
if check_hwcodec_config() {
let best = HwEncoder::best();
(
best.h264.as_ref().map_or(false, |c| c.score > 0),
best.h265.as_ref().map_or(false, |c| c.score > 0),
)
} else {
(false, false)
}
#[cfg(not(feature = "hwcodec"))]
(false, false)
}
}
impl Decoder {
pub fn video_codec_state() -> VideoCodecState {
// video_codec_state is mainted by creation and destruction of Decoder.
// It has been ensured to use after Decoder's creation.
pub fn video_codec_state(_id: &str) -> VideoCodecState {
#[cfg(feature = "hwcodec")]
if check_hwcodec_config() {
return MY_DECODER_STATE.lock().unwrap().clone();
let best = HwDecoder::best();
VideoCodecState {
score_vpx: SCORE_VPX,
score_h264: best.h264.map_or(0, |c| c.score),
score_h265: best.h265.map_or(0, |c| c.score),
perfer: Self::codec_preference(_id).into(),
..Default::default()
}
} else {
return VideoCodecState {
ScoreVpx: SCORE_VPX,
score_vpx: SCORE_VPX,
..Default::default()
};
}
#[cfg(not(feature = "hwcodec"))]
VideoCodecState {
ScoreVpx: SCORE_VPX,
score_vpx: SCORE_VPX,
..Default::default()
}
}
pub fn new(config: DecoderCfg) -> Decoder {
let vpx = VpxDecoder::new(config.vpx).unwrap();
let decoder = Decoder {
Decoder {
vpx,
#[cfg(feature = "hwcodec")]
hw: HwDecoder::new_decoders(),
#[cfg(feature = "hwcodec")]
i420: vec![],
};
#[cfg(feature = "hwcodec")]
{
let mut state = MY_DECODER_STATE.lock().unwrap();
state.ScoreVpx = SCORE_VPX;
state.ScoreH264 = decoder.hw.h264.as_ref().map_or(0, |d| d.info.score);
state.ScoreH265 = decoder.hw.h265.as_ref().map_or(0, |d| d.info.score);
}
decoder
}
pub fn handle_video_frame(
@ -316,6 +344,23 @@ impl Decoder {
}
return Ok(ret);
}
#[cfg(feature = "hwcodec")]
fn codec_preference(id: &str) -> PerferCodec {
let codec = PeerConfig::load(id)
.options
.get("codec-preference")
.map_or("".to_owned(), |c| c.to_owned());
if codec == "vp9" {
PerferCodec::VPX
} else if codec == "h264" {
PerferCodec::H264
} else if codec == "h265" {
PerferCodec::H265
} else {
PerferCodec::Auto
}
}
}
#[cfg(feature = "hwcodec")]

View File

@ -246,6 +246,7 @@ pub unsafe fn nv12_to_i420(
#[cfg(feature = "hwcodec")]
pub mod hw {
use hbb_common::{anyhow::anyhow, ResultType};
#[cfg(target_os = "windows")]
use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat};
pub fn hw_bgra_to_i420(
@ -381,6 +382,8 @@ pub mod hw {
src_stride_y: usize,
src_stride_uv: usize,
dst: &mut Vec<u8>,
_i420: &mut Vec<u8>,
_align: usize,
) -> ResultType<()> {
dst.resize(width * height * 4, 0);
unsafe {

View File

@ -123,40 +123,11 @@ impl EncoderApi for HwEncoder {
}
impl HwEncoder {
/// Get best encoders.
///
/// # Parameter
/// `force_reset`: force to refresh config.
/// `write`: write to config file.
///
/// # Return
/// `CodecInfos`: infos.
/// `bool`: whether the config is refreshed.
pub fn best(force_reset: bool, write: bool) -> (CodecInfos, bool) {
let config = get_config(CFG_KEY_ENCODER);
if !force_reset && config.is_ok() {
(config.unwrap(), false)
} else {
let ctx = EncodeContext {
name: String::from(""),
width: 1920,
height: 1080,
pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _,
bitrate: 0,
timebase: DEFAULT_TIME_BASE,
gop: DEFAULT_GOP,
quality: DEFAULT_HW_QUALITY,
rc: DEFAULT_RC,
};
let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx));
if write {
set_config(CFG_KEY_ENCODER, &encoders)
.map_err(|e| log::error!("{:?}", e))
.ok();
}
(encoders, true)
}
pub fn best() -> CodecInfos {
get_config(CFG_KEY_ENCODER).unwrap_or(CodecInfos {
h264: None,
h265: None,
})
}
pub fn current_name() -> Arc<Mutex<Option<String>>> {
@ -207,24 +178,15 @@ pub struct HwDecoders {
}
impl HwDecoder {
/// See HwEncoder::best
fn best(force_reset: bool, write: bool) -> (CodecInfos, bool) {
let config = get_config(CFG_KEY_DECODER);
if !force_reset && config.is_ok() {
(config.unwrap(), false)
} else {
let decoders = CodecInfo::score(Decoder::avaliable_decoders());
if write {
set_config(CFG_KEY_DECODER, &decoders)
.map_err(|e| log::error!("{:?}", e))
.ok();
}
(decoders, true)
}
pub fn best() -> CodecInfos {
get_config(CFG_KEY_DECODER).unwrap_or(CodecInfos {
h264: None,
h265: None,
})
}
pub fn new_decoders() -> HwDecoders {
let (best, _) = HwDecoder::best(false, true);
let best = HwDecoder::best();
let mut h264: Option<HwDecoder> = None;
let mut h265: Option<HwDecoder> = None;
let mut fail = false;
@ -242,7 +204,7 @@ impl HwDecoder {
}
}
if fail {
HwDecoder::best(true, true);
check_config_process(true);
}
HwDecoders { h264, h265 }
}
@ -314,31 +276,52 @@ fn get_config(k: &str) -> ResultType<CodecInfos> {
}
}
fn set_config(k: &str, v: &CodecInfos) -> ResultType<()> {
match v.serialize() {
Ok(v) => {
let mut config = HwCodecConfig::load();
config.options.insert(k.to_owned(), v);
config.store();
Ok(())
}
Err(_) => Err(anyhow!("Failed to set config:{}", k)),
}
}
pub fn check_config() {
let (encoders, update_encoders) = HwEncoder::best(false, false);
let (decoders, update_decoders) = HwDecoder::best(false, false);
if update_encoders || update_decoders {
if let Ok(encoders) = encoders.serialize() {
if let Ok(decoders) = decoders.serialize() {
let mut config = HwCodecConfig::load();
config.options.insert(CFG_KEY_ENCODER.to_owned(), encoders);
config.options.insert(CFG_KEY_DECODER.to_owned(), decoders);
config.store();
let ctx = EncodeContext {
name: String::from(""),
width: 1920,
height: 1080,
pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _,
bitrate: 0,
timebase: DEFAULT_TIME_BASE,
gop: DEFAULT_GOP,
quality: DEFAULT_HW_QUALITY,
rc: DEFAULT_RC,
};
let encoders = CodecInfo::score(Encoder::avaliable_encoders(ctx));
let decoders = CodecInfo::score(Decoder::avaliable_decoders());
if let Ok(old_encoders) = get_config(CFG_KEY_ENCODER) {
if let Ok(old_decoders) = get_config(CFG_KEY_DECODER) {
if encoders == old_encoders && decoders == old_decoders {
return;
}
}
log::error!("Failed to serialize codec info");
}
if let Ok(encoders) = encoders.serialize() {
if let Ok(decoders) = decoders.serialize() {
let mut config = HwCodecConfig::load();
config.options.insert(CFG_KEY_ENCODER.to_owned(), encoders);
config.options.insert(CFG_KEY_DECODER.to_owned(), decoders);
config.store();
return;
}
}
log::error!("Failed to serialize codec info");
}
pub fn check_config_process(force_reset: bool) {
if force_reset {
HwCodecConfig::remove();
}
if let Ok(exe) = std::env::current_exe() {
std::thread::spawn(move || {
std::process::Command::new(exe)
.arg("--check-hwcodec-config")
.status()
.ok()
});
};
}

View File

@ -784,6 +784,7 @@ pub struct LoginConfigHandler {
pub conn_id: i32,
features: Option<Features>,
session_id: u64,
pub supported_encoding: Option<(bool, bool)>,
}
impl Deref for LoginConfigHandler {
@ -808,6 +809,7 @@ impl LoginConfigHandler {
self.remember = !config.password.is_empty();
self.config = config;
self.session_id = rand::random();
self.supported_encoding = None;
}
pub fn should_auto_login(&self) -> String {
@ -958,8 +960,7 @@ impl LoginConfigHandler {
msg.disable_clipboard = BoolOption::Yes.into();
n += 1;
}
// TODO: add option
let state = Decoder::video_codec_state();
let state = Decoder::video_codec_state(&self.id);
msg.video_codec_state = hbb_common::protobuf::MessageField::some(state);
n += 1;
@ -1111,6 +1112,10 @@ impl LoginConfigHandler {
self.conn_id = pi.conn_id;
// no matter if change, for update file time
self.save_config(config);
#[cfg(feature = "hwcodec")]
{
self.supported_encoding = Some((pi.encoding.h264, pi.encoding.h265));
}
}
pub fn get_remote_dir(&self) -> String {
@ -1163,6 +1168,18 @@ impl LoginConfigHandler {
msg_out.set_login_request(lr);
msg_out
}
pub fn change_prefer_codec(&self) -> Message {
let state = scrap::codec::Decoder::video_codec_state(&self.id);
let mut misc = Misc::new();
misc.set_option(OptionMessage {
video_codec_state: hbb_common::protobuf::MessageField::some(state),
..Default::default()
});
let mut msg_out = Message::new();
msg_out.set_misc(misc);
msg_out
}
}
pub enum MediaData {

View File

@ -321,6 +321,14 @@ pub async fn start_server(is_server: bool) {
log::info!("DISPLAY={:?}", std::env::var("DISPLAY"));
log::info!("XAUTHORITY={:?}", std::env::var("XAUTHORITY"));
}
#[cfg(feature = "hwcodec")]
{
use std::sync::Once;
static ONCE: Once = Once::new();
ONCE.call_once(|| {
scrap::hwcodec::check_config_process(false);
})
}
if is_server {
std::thread::spawn(move || {
@ -329,15 +337,6 @@ pub async fn start_server(is_server: bool) {
std::process::exit(-1);
}
});
#[cfg(feature = "hwcodec")]
if let Ok(exe) = std::env::current_exe() {
std::thread::spawn(move || {
std::process::Command::new(exe)
.arg("--check-hwcodec-config")
.status()
.ok()
});
}
#[cfg(windows)]
crate::platform::windows::bootstrap();
input_service::fix_key_down_timeout_loop();

View File

@ -635,6 +635,16 @@ impl Connection {
pi.hostname = MOBILE_INFO2.lock().unwrap().clone();
pi.platform = "Android".into();
}
#[cfg(feature = "hwcodec")]
{
let (h264, h265) = scrap::codec::Encoder::supported_encoding();
pi.encoding = Some(SupportedEncoding {
h264,
h265,
..Default::default()
})
.into();
}
if self.port_forward_socket.is_some() {
let mut msg_out = Message::new();
@ -1351,6 +1361,12 @@ impl Connection {
}
}
}
if let Some(q) = o.video_codec_state.clone().take() {
scrap::codec::Encoder::update_video_encoder(
self.inner.id(),
scrap::codec::EncoderUpdate::State(q),
);
}
}
async fn on_close(&mut self, reason: &str, lock: bool) {

View File

@ -435,6 +435,7 @@ fn run(sp: GenericService) -> ResultType<()> {
let mut try_gdi = 1;
#[cfg(windows)]
log::info!("gdi: {}", c.is_gdi());
let codec_name = Encoder::current_hw_encoder_name();
while sp.ok() {
#[cfg(windows)]
@ -460,6 +461,9 @@ fn run(sp: GenericService) -> ResultType<()> {
*SWITCH.lock().unwrap() = true;
bail!("SWITCH");
}
if codec_name != Encoder::current_hw_encoder_name() {
bail!("SWITCH");
}
check_privacy_mode_changed(&sp, c.privacy_mode_id)?;
#[cfg(windows)]
{

View File

@ -94,3 +94,4 @@ span#fullscreen.active {
button:disabled {
opacity: 0.3;
}

View File

@ -145,6 +145,9 @@ class Header: Reactor.Component {
}
function renderDisplayPop() {
var codecs = handler.supported_hwcodec();
var show_codec = handler.has_hwcodec() && (codecs[0] || codecs[1]);
return <popup>
<menu.context #display-options>
<li #adjust-window style="display:none">{translate('Adjust Window')}</li>
@ -157,6 +160,13 @@ class Header: Reactor.Component {
<li #balanced type="image-quality"><span>{svg_checkmark}</span>{translate('Balanced')}</li>
<li #low type="image-quality"><span>{svg_checkmark}</span>{translate('Optimize reaction time')}</li>
<li #custom type="image-quality"><span>{svg_checkmark}</span>{translate('Custom')}</li>
{show_codec ? <div>
<div .separator />
<li #auto type="codec-preference"><span>{svg_checkmark}</span>Auto</li>
<li #vp9 type="codec-preference"><span>{svg_checkmark}</span>VP9</li>
{codecs[0] ? <li #h264 type="codec-preference"><span>{svg_checkmark}</span>H264</li> : ""}
{codecs[1] ? <li #h265 type="codec-preference"><span>{svg_checkmark}</span>H265</li> : ""}
</div> : ""}
<div .separator />
<li #show-remote-cursor .toggle-option><span>{svg_checkmark}</span>{translate('Show remote cursor')}</li>
<li #show-quality-monitor .toggle-option><span>{svg_checkmark}</span>{translate('Show quality monitor')}</li>
@ -311,7 +321,7 @@ class Header: Reactor.Component {
}
}
event click $(menu#display-options>li) (_, me) {
event click $(menu#display-options li) (_, me) {
if (me.id == "custom") {
handle_custom_image_quality();
} else if (me.id == "privacy-mode") {
@ -328,6 +338,9 @@ class Header: Reactor.Component {
} else if (type == "view-style") {
handler.save_view_style(me.id);
adaptDisplay();
} else if (type == "codec-preference") {
handler.set_option("codec-preference", me.id);
handler.change_prefer_codec();
}
toggleMenuState();
}
@ -355,7 +368,10 @@ function toggleMenuState() {
var s = handler.get_view_style();
if (!s) s = "original";
values.push(s);
for (var el in $$(menu#display-options>li)) {
var c = handler.get_option("codec-preference");
if (!c) c = "auto";
values.push(c);
for (var el in $$(menu#display-options li)) {
el.attributes.toggleClass("selected", values.indexOf(el.id) >= 0);
}
for (var id in ["show-remote-cursor", "show-quality-monitor", "disable-audio", "enable-file-transfer", "disable-clipboard", "lock-after-session-end"]) {

View File

@ -231,6 +231,9 @@ impl sciter::EventHandler for Handler {
fn get_remember();
fn peer_platform();
fn set_write_override(i32, i32, bool, bool, bool);
fn has_hwcodec();
fn supported_hwcodec();
fn change_prefer_codec();
}
}
@ -595,6 +598,42 @@ impl Handler {
true
}
fn has_hwcodec(&self) -> bool {
#[cfg(not(feature = "hwcodec"))]
return false;
#[cfg(feature = "hwcodec")]
return true;
}
fn supported_hwcodec(&self) -> Value {
#[cfg(feature = "hwcodec")]
{
let mut v = Value::array(0);
let decoder = scrap::codec::Decoder::video_codec_state(&self.id);
let mut h264 = decoder.score_h264 > 0;
let mut h265 = decoder.score_h265 > 0;
if let Some((encoding_264, encoding_265)) = self.lc.read().unwrap().supported_encoding {
h264 = h264 && encoding_264;
h265 = h265 && encoding_265;
}
v.push(h264);
v.push(h265);
v
}
#[cfg(not(feature = "hwcodec"))]
{
let mut v = Value::array(0);
v.push(false);
v.push(false);
v
}
}
fn change_prefer_codec(&self) {
let msg = self.lc.write().unwrap().change_prefer_codec();
self.send(Data::Message(msg));
}
fn t(&self, name: String) -> String {
crate::client::translate(name)
}