hongshaorou/rustdesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add ffmpeg mediacodec h264/h265 encode (#8028)

Browse Source 
* Check available when app start from kotlin via get codec info
* For latency free, repeat encode 10 frame at most when capture return WouldBlock
* For changing quality, kotlin support but jni doesn't support, rerun video service when quality is manualy
  changed
* 3 or 6 times bitrate for mediacodec because its quality is poor

Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages 2024-05-13 12:39:04 +08:00 committed by GitHub
parent 4c99b8c70e
commit a7499c2de8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
13 changed files with 380 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
Cargo.lock generated
View File

@ -3037,8 +3037,8 @@ checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4"
[[package]] [[package]]
name = "hwcodec" name = "hwcodec"
version = "0.4.9" version = "0.4.10"
source = "git+https://github.com/21pages/hwcodec#7a52282267cb6aadbcd74c132bd4ecd43ab6f505" source = "git+https://github.com/21pages/hwcodec#9cb895fdaea198dd72bd75980109dbd05a059a60"
dependencies = [ dependencies = [
"bindgen 0.59.2", "bindgen 0.59.2",
"cc", "cc",

89
flutter/android/app/src/main/kotlin/com/carriez/flutter_hbb/MainActivity.kt
View File

@ -7,6 +7,8 @@ package com.carriez.flutter_hbb
* Inspired by [droidVNC-NG] https://github.com/bk138/droidVNC-NG * Inspired by [droidVNC-NG] https://github.com/bk138/droidVNC-NG
*/ */
import ffi.FFI
import android.content.ComponentName import android.content.ComponentName
import android.content.Context import android.content.Context
import android.content.Intent import android.content.Intent
@ -15,10 +17,20 @@ import android.os.Build
import android.os.IBinder import android.os.IBinder
import android.util.Log import android.util.Log
import android.view.WindowManager import android.view.WindowManager
import android.media.MediaCodecInfo
import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface
import android.media.MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420SemiPlanar
import android.media.MediaCodecList
import android.media.MediaFormat
import android.util.DisplayMetrics
import androidx.annotation.RequiresApi
import org.json.JSONArray
import org.json.JSONObject
import com.hjq.permissions.XXPermissions import com.hjq.permissions.XXPermissions
import io.flutter.embedding.android.FlutterActivity import io.flutter.embedding.android.FlutterActivity
import io.flutter.embedding.engine.FlutterEngine import io.flutter.embedding.engine.FlutterEngine
import io.flutter.plugin.common.MethodChannel import io.flutter.plugin.common.MethodChannel
import kotlin.concurrent.thread
class MainActivity : FlutterActivity() { class MainActivity : FlutterActivity() {
@ -42,6 +54,7 @@ class MainActivity : FlutterActivity() {
channelTag channelTag
) )
initFlutterChannel(flutterMethodChannel!!) initFlutterChannel(flutterMethodChannel!!)
thread { setCodecInfo() }
} }
override fun onResume() { override fun onResume() {
@ -223,4 +236,80 @@ class MainActivity : FlutterActivity() {
} }
} }
} }
private fun setCodecInfo() {
val codecList = MediaCodecList(MediaCodecList.REGULAR_CODECS)
val codecs = codecList.codecInfos
val codecArray = JSONArray()
val windowManager = getSystemService(Context.WINDOW_SERVICE) as WindowManager
var w = 0
var h = 0
@Suppress("DEPRECATION")
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
val m = windowManager.maximumWindowMetrics
w = m.bounds.width()
h = m.bounds.height()
} else {
val dm = DisplayMetrics()
windowManager.defaultDisplay.getRealMetrics(dm)
w = dm.widthPixels
h = dm.heightPixels
}
codecs.forEach { codec ->
val codecObject = JSONObject()
codecObject.put("name", codec.name)
codecObject.put("is_encoder", codec.isEncoder)
var hw: Boolean? = null;
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
hw = codec.isHardwareAccelerated
} else {
if (listOf("OMX.google.", "OMX.SEC.", "c2.android").any { codec.name.startsWith(it) }) {
hw = false
}
}
codecObject.put("hw", hw)
var mime_type = ""
codec.supportedTypes.forEach { type ->
if (listOf("video/avc", "video/hevc", "video/x-vnd.on2.vp8", "video/x-vnd.on2.vp9", "video/av01").contains(type)) {
mime_type = type;
}
}
if (mime_type.isNotEmpty()) {
codecObject.put("mime_type", mime_type)
val caps = codec.getCapabilitiesForType(mime_type)
var usable = true;
if (codec.isEncoder) {
if (!caps.videoCapabilities.isSizeSupported(w,h) || !caps.videoCapabilities.isSizeSupported(h,w)) {
usable = false
}
}
codecObject.put("min_width", caps.videoCapabilities.supportedWidths.lower)
codecObject.put("max_width", caps.videoCapabilities.supportedWidths.upper)
codecObject.put("min_height", caps.videoCapabilities.supportedHeights.lower)
codecObject.put("max_height", caps.videoCapabilities.supportedHeights.upper)
val surface = caps.colorFormats.contains(COLOR_FormatSurface);
codecObject.put("surface", surface)
val nv12 = caps.colorFormats.contains(COLOR_FormatYUV420SemiPlanar)
codecObject.put("nv12", nv12)
if (!(nv12 || surface)) {
usable = false
}
codecObject.put("min_bitrate", caps.videoCapabilities.bitrateRange.lower / 1000)
codecObject.put("max_bitrate", caps.videoCapabilities.bitrateRange.upper / 1000)
if (!codec.isEncoder) {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
codecObject.put("low_latency", caps.isFeatureSupported(MediaCodecInfo.CodecCapabilities.FEATURE_LowLatency))
}
}
if (usable) {
codecArray.put(codecObject)
}
}
}
val result = JSONObject()
result.put("version", Build.VERSION.SDK_INT)
result.put("codecs", codecArray)
FFI.setCodecInfo(result.toString())
}
} }

1
flutter/android/app/src/main/kotlin/ffi.kt
View File

@ -18,4 +18,5 @@ object FFI {
external fun translateLocale(localeName: String, input: String): String external fun translateLocale(localeName: String, input: String): String
external fun refreshScreen() external fun refreshScreen()
external fun setFrameRawEnable(name: String, value: Boolean) external fun setFrameRawEnable(name: String, value: Boolean)
external fun setCodecInfo(info: String)
} }

24
flutter/lib/mobile/pages/settings_page.dart
View File

@ -44,6 +44,7 @@ class _SettingsState extends State<SettingsPage> with WidgetsBindingObserver {
var _onlyWhiteList = false; var _onlyWhiteList = false;
var _enableDirectIPAccess = false; var _enableDirectIPAccess = false;
var _enableRecordSession = false; var _enableRecordSession = false;
var _enableHardwareCodec = false;
var _autoRecordIncomingSession = false; var _autoRecordIncomingSession = false;
var _allowAutoDisconnect = false; var _allowAutoDisconnect = false;
var _localIP = ""; var _localIP = "";
@ -120,6 +121,13 @@ class _SettingsState extends State<SettingsPage> with WidgetsBindingObserver {
_enableRecordSession = enableRecordSession; _enableRecordSession = enableRecordSession;
} }
final enableHardwareCodec = option2bool(
'enable-hwcodec', await bind.mainGetOption(key: 'enable-hwcodec'));
if (_enableHardwareCodec != enableHardwareCodec) {
update = true;
_enableHardwareCodec = enableHardwareCodec;
}
final autoRecordIncomingSession = option2bool( final autoRecordIncomingSession = option2bool(
'allow-auto-record-incoming', 'allow-auto-record-incoming',
await bind.mainGetOption(key: 'allow-auto-record-incoming')); await bind.mainGetOption(key: 'allow-auto-record-incoming'));
@ -513,6 +521,22 @@ class _SettingsState extends State<SettingsPage> with WidgetsBindingObserver {
}, },
) )
]), ]),
if (isAndroid)
SettingsSection(title: Text(translate('Hardware Codec')), tiles: [
SettingsTile.switchTile(
title: Text(translate('Enable hardware codec')),
initialValue: _enableHardwareCodec,
onToggle: (v) async {
await bind.mainSetOption(
key: "enable-hwcodec", value: v ? "" : "N");
final newValue =
await bind.mainGetOption(key: "enable-hwcodec") != "N";
setState(() {
_enableHardwareCodec = newValue;
});
},
),
]),
if (isAndroid && !outgoingOnly) if (isAndroid && !outgoingOnly)
SettingsSection( SettingsSection(
title: Text(translate("Recording")), title: Text(translate("Recording")),

1
libs/scrap/examples/benchmark.rs
View File

@ -272,6 +272,7 @@ mod hw {
let mut encoder = HwRamEncoder::new( let mut encoder = HwRamEncoder::new(
EncoderCfg::HWRAM(HwRamEncoderConfig { EncoderCfg::HWRAM(HwRamEncoderConfig {
name: info.name.clone(), name: info.name.clone(),
mc_name: None,
width, width,
height, height,
quality, quality,

46
libs/scrap/src/android/ffi.rs
View File

@ -10,6 +10,7 @@ use jni::{
use jni::errors::{Error as JniError, Result as JniResult}; use jni::errors::{Error as JniError, Result as JniResult};
use lazy_static::lazy_static; use lazy_static::lazy_static;
use serde::Deserialize;
use std::ops::Not; use std::ops::Not;
use std::sync::atomic::{AtomicPtr, Ordering::SeqCst}; use std::sync::atomic::{AtomicPtr, Ordering::SeqCst};
use std::sync::{Mutex, RwLock}; use std::sync::{Mutex, RwLock};
@ -20,6 +21,7 @@ lazy_static! {
static ref VIDEO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("video", MAX_VIDEO_FRAME_TIMEOUT)); static ref VIDEO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("video", MAX_VIDEO_FRAME_TIMEOUT));
static ref AUDIO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("audio", MAX_AUDIO_FRAME_TIMEOUT)); static ref AUDIO_RAW: Mutex<FrameRaw> = Mutex::new(FrameRaw::new("audio", MAX_AUDIO_FRAME_TIMEOUT));
static ref NDK_CONTEXT_INITED: Mutex<bool> = Default::default(); static ref NDK_CONTEXT_INITED: Mutex<bool> = Default::default();
static ref MEDIA_CODEC_INFOS: RwLock<Option<MediaCodecInfos>> = RwLock::new(None);
} }
const MAX_VIDEO_FRAME_TIMEOUT: Duration = Duration::from_millis(100); const MAX_VIDEO_FRAME_TIMEOUT: Duration = Duration::from_millis(100);
@ -154,6 +156,50 @@ pub extern "system" fn Java_ffi_FFI_init(env: JNIEnv, _class: JClass, ctx: JObje
} }
} }
#[derive(Debug, Deserialize, Clone)]
pub struct MediaCodecInfo {
pub name: String,
pub is_encoder: bool,
#[serde(default)]
pub hw: Option<bool>, // api 29+
pub mime_type: String,
pub surface: bool,
pub nv12: bool,
#[serde(default)]
pub low_latency: Option<bool>, // api 30+, decoder
pub min_bitrate: u32,
pub max_bitrate: u32,
pub min_width: usize,
pub max_width: usize,
pub min_height: usize,
pub max_height: usize,
}
#[derive(Debug, Deserialize, Clone)]
pub struct MediaCodecInfos {
pub version: usize,
pub codecs: Vec<MediaCodecInfo>,
}
#[no_mangle]
pub extern "system" fn Java_ffi_FFI_setCodecInfo(env: JNIEnv, _class: JClass, info: JString) {
let mut env = env;
if let Ok(info) = env.get_string(&info) {
let info: String = info.into();
if let Ok(infos) = serde_json::from_str::<MediaCodecInfos>(&info) {
*MEDIA_CODEC_INFOS.write().unwrap() = Some(infos);
}
}
}
pub fn get_codec_info() -> Option<MediaCodecInfos> {
MEDIA_CODEC_INFOS.read().unwrap().as_ref().cloned()
}
pub fn clear_codec_info() {
*MEDIA_CODEC_INFOS.write().unwrap() = None;
}
pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> { pub fn call_main_service_pointer_input(kind: &str, mask: i32, x: i32, y: i32) -> JniResult<()> {
if let (Some(jvm), Some(ctx)) = ( if let (Some(jvm), Some(ctx)) = (
JVM.read().unwrap().as_ref(), JVM.read().unwrap().as_ref(),

8
libs/scrap/src/common/aom.rs
View File

@ -296,6 +296,14 @@ impl EncoderApi for AomEncoder {
fn support_abr(&self) -> bool { fn support_abr(&self) -> bool {
true true
} }
fn support_changing_quality(&self) -> bool {
true
}
fn latency_free(&self) -> bool {
true
}
} }
impl AomEncoder { impl AomEncoder {

27
libs/scrap/src/common/codec.rs
View File

@ -70,6 +70,10 @@ pub trait EncoderApi {
fn bitrate(&self) -> u32; fn bitrate(&self) -> u32;
fn support_abr(&self) -> bool; fn support_abr(&self) -> bool;
fn support_changing_quality(&self) -> bool;
fn latency_free(&self) -> bool;
} }
pub struct Encoder { pub struct Encoder {
@ -138,6 +142,9 @@ impl Encoder {
}), }),
Err(e) => { Err(e) => {
log::error!("new hw encoder failed: {e:?}, clear config"); log::error!("new hw encoder failed: {e:?}, clear config");
#[cfg(target_os = "android")]
crate::android::ffi::clear_codec_info();
#[cfg(not(target_os = "android"))]
hbb_common::config::HwCodecConfig::clear_ram(); hbb_common::config::HwCodecConfig::clear_ram();
Self::update(EncodingUpdate::Check); Self::update(EncodingUpdate::Check);
*ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9; *ENCODE_CODEC_FORMAT.lock().unwrap() = CodecFormat::VP9;
@ -346,7 +353,14 @@ impl Encoder {
EncoderCfg::AOM(_) => CodecFormat::AV1, EncoderCfg::AOM(_) => CodecFormat::AV1,
#[cfg(feature = "hwcodec")] #[cfg(feature = "hwcodec")]
EncoderCfg::HWRAM(hw) => { EncoderCfg::HWRAM(hw) => {
if hw.name.to_lowercase().contains("h264") { let name = hw.name.to_lowercase();
if name.contains("vp8") {
CodecFormat::VP8
} else if name.contains("vp9") {
CodecFormat::VP9
} else if name.contains("av1") {
CodecFormat::AV1
} else if name.contains("h264") {
CodecFormat::H264 CodecFormat::H264
} else { } else {
CodecFormat::H265 CodecFormat::H265
@ -817,7 +831,7 @@ impl Decoder {
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))] #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
pub fn enable_hwcodec_option() -> bool { pub fn enable_hwcodec_option() -> bool {
if cfg!(windows) || cfg!(target_os = "linux") || cfg!(feature = "mediacodec") { if cfg!(windows) || cfg!(target_os = "linux") || cfg!(target_os = "android") {
if let Some(v) = Config2::get().options.get("enable-hwcodec") { if let Some(v) = Config2::get().options.get("enable-hwcodec") {
return v != "N"; return v != "N";
} }
@ -847,6 +861,15 @@ impl Default for Quality {
} }
} }
impl Quality {
pub fn is_custom(&self) -> bool {
match self {
Quality::Custom(_) => true,
_ => false,
}
}
}
pub fn base_bitrate(width: u32, height: u32) -> u32 { pub fn base_bitrate(width: u32, height: u32) -> u32 {
#[allow(unused_mut)] #[allow(unused_mut)]
let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9 let mut base_bitrate = ((width * height) / 1000) as u32; // same as 1.1.9

139
libs/scrap/src/common/hwcodec.rs
View File

@ -29,11 +29,15 @@ const DEFAULT_PIXFMT: AVPixelFormat = AVPixelFormat::AV_PIX_FMT_NV12;
pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30]; pub const DEFAULT_TIME_BASE: [i32; 2] = [1, 30];
const DEFAULT_GOP: i32 = i32::MAX; const DEFAULT_GOP: i32 = i32::MAX;
const DEFAULT_HW_QUALITY: Quality = Quality_Default; const DEFAULT_HW_QUALITY: Quality = Quality_Default;
const DEFAULT_RC: RateControl = RC_DEFAULT; #[cfg(target_os = "android")]
const DEFAULT_RC: RateControl = RC_VBR; // android cbr poor quality
#[cfg(not(target_os = "android"))]
const DEFAULT_RC: RateControl = RC_CBR;
#[derive(Debug, Clone)] #[derive(Debug, Clone)]
pub struct HwRamEncoderConfig { pub struct HwRamEncoderConfig {
pub name: String, pub name: String,
pub mc_name: Option<String>,
pub width: usize, pub width: usize,
pub height: usize, pub height: usize,
pub quality: Q, pub quality: Q,
@ -57,20 +61,22 @@ impl EncoderApi for HwRamEncoder {
{ {
match cfg { match cfg {
EncoderCfg::HWRAM(config) => { EncoderCfg::HWRAM(config) => {
let b = Self::convert_quality(config.quality); let b = Self::convert_quality(&config.name, config.quality);
let base_bitrate = base_bitrate(config.width as _, config.height as _); let base_bitrate = base_bitrate(config.width as _, config.height as _);
let mut bitrate = base_bitrate * b / 100; let mut bitrate = base_bitrate * b / 100;
if base_bitrate <= 0 { if base_bitrate <= 0 {
bitrate = base_bitrate; bitrate = base_bitrate;
} }
bitrate = Self::check_bitrate_range(&config.name, bitrate);
let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32; let gop = config.keyframe_interval.unwrap_or(DEFAULT_GOP as _) as i32;
let ctx = EncodeContext { let ctx = EncodeContext {
name: config.name.clone(), name: config.name.clone(),
mc_name: config.mc_name.clone(),
width: config.width as _, width: config.width as _,
height: config.height as _, height: config.height as _,
pixfmt: DEFAULT_PIXFMT, pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _, align: HW_STRIDE_ALIGN as _,
bitrate: bitrate as i32 * 1000, kbs: bitrate as i32,
timebase: DEFAULT_TIME_BASE, timebase: DEFAULT_TIME_BASE,
gop, gop,
quality: DEFAULT_HW_QUALITY, quality: DEFAULT_HW_QUALITY,
@ -166,10 +172,11 @@ impl EncoderApi for HwRamEncoder {
} }
fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> { fn set_quality(&mut self, quality: crate::codec::Quality) -> ResultType<()> {
let b = Self::convert_quality(quality); let b = Self::convert_quality(&self.name, quality);
let bitrate = base_bitrate(self.width as _, self.height as _) * b / 100; let mut bitrate = base_bitrate(self.width as _, self.height as _) * b / 100;
if bitrate > 0 { if bitrate > 0 {
self.encoder.set_bitrate((bitrate * 1000) as _).ok(); bitrate = Self::check_bitrate_range(&self.name, bitrate);
self.encoder.set_bitrate(bitrate as _).ok();
self.bitrate = bitrate; self.bitrate = bitrate;
} }
Ok(()) Ok(())
@ -180,7 +187,19 @@ impl EncoderApi for HwRamEncoder {
} }
fn support_abr(&self) -> bool { fn support_abr(&self) -> bool {
["qsv", "vaapi"].iter().all(|&x| !self.name.contains(x)) ["qsv", "vaapi", "mediacodec"]
.iter()
.all(|&x| !self.name.contains(x))
}
fn support_changing_quality(&self) -> bool {
["vaapi", "mediacodec"]
.iter()
.all(|&x| !self.name.contains(x))
}
fn latency_free(&self) -> bool {
!self.name.contains("mediacodec")
} }
} }
@ -217,14 +236,42 @@ impl HwRamEncoder {
} }
} }
pub fn convert_quality(quality: crate::codec::Quality) -> u32 { pub fn convert_quality(name: &str, quality: crate::codec::Quality) -> u32 {
use crate::codec::Quality; use crate::codec::Quality;
match quality { let quality = match quality {
Quality::Best => 150, Quality::Best => 150,
Quality::Balanced => 100, Quality::Balanced => 100,
Quality::Low => 50, Quality::Low => 50,
Quality::Custom(b) => b, Quality::Custom(b) => b,
};
let factor = if name.contains("mediacodec") {
if name.contains("h264") {
6
} else {
3
} }
} else {
1
};
quality * factor
}
pub fn check_bitrate_range(name: &str, bitrate: u32) -> u32 {
#[cfg(target_os = "android")]
if name.contains("mediacodec") {
let info = crate::android::ffi::get_codec_info();
if let Some(info) = info {
if let Some(codec) = info.codecs.iter().find(|c| c.name == name && c.is_encoder) {
if bitrate > codec.max_bitrate {
return codec.max_bitrate;
}
if bitrate < codec.min_bitrate {
return codec.min_bitrate;
}
}
}
}
bitrate
} }
} }
@ -285,7 +332,10 @@ impl HwRamDecoder {
match Decoder::new(ctx) { match Decoder::new(ctx) {
Ok(decoder) => Ok(HwRamDecoder { decoder, info }), Ok(decoder) => Ok(HwRamDecoder { decoder, info }),
Err(_) => { Err(_) => {
HwCodecConfig::clear_ram(); #[cfg(target_os = "android")]
crate::android::ffi::clear_codec_info();
#[cfg(not(target_os = "android"))]
hbb_common::config::HwCodecConfig::clear_ram();
Err(anyhow!(format!("Failed to create decoder"))) Err(anyhow!(format!("Failed to create decoder")))
} }
} }
@ -363,20 +413,87 @@ struct Available {
} }
fn get_config() -> ResultType<Available> { fn get_config() -> ResultType<Available> {
#[cfg(target_os = "android")]
{
let info = crate::android::ffi::get_codec_info();
struct T {
name_prefix: &'static str,
data_format: DataFormat,
}
let ts = vec![
T {
name_prefix: "h264",
data_format: DataFormat::H264,
},
T {
name_prefix: "hevc",
data_format: DataFormat::H265,
},
];
let mut e = vec![];
if let Some(info) = info {
ts.iter().for_each(|t| {
let codecs: Vec<_> = info
.codecs
.iter()
.filter(|c| {
c.is_encoder
&& c.mime_type.as_str() == get_mime_type(t.data_format)
&& c.nv12
})
.collect();
log::debug!("available {:?} encoders: {codecs:?}", t.data_format);
let mut best = None;
if let Some(c) = codecs.iter().find(|c| c.hw == Some(true)) {
best = Some(c.name.clone());
} else if let Some(c) = codecs.iter().find(|c| c.hw == None) {
best = Some(c.name.clone());
} else if let Some(c) = codecs.first() {
best = Some(c.name.clone());
}
if let Some(best) = best {
e.push(CodecInfo {
name: format!("{}_mediacodec", t.name_prefix),
mc_name: Some(best),
format: t.data_format,
hwdevice: hwcodec::ffmpeg::AVHWDeviceType::AV_HWDEVICE_TYPE_NONE,
priority: 0,
});
}
});
}
log::debug!("e: {e:?}");
Ok(Available { e, d: vec![] })
}
#[cfg(not(target_os = "android"))]
{
match serde_json::from_str(&HwCodecConfig::load().ram) { match serde_json::from_str(&HwCodecConfig::load().ram) {
Ok(v) => Ok(v), Ok(v) => Ok(v),
Err(e) => Err(anyhow!("Failed to get config:{e:?}")), Err(e) => Err(anyhow!("Failed to get config:{e:?}")),
} }
} }
}
#[cfg(target_os = "android")]
fn get_mime_type(codec: DataFormat) -> &'static str {
match codec {
DataFormat::VP8 => "video/x-vnd.on2.vp8",
DataFormat::VP9 => "video/x-vnd.on2.vp9",
DataFormat::AV1 => "video/av01",
DataFormat::H264 => "video/avc",
DataFormat::H265 => "video/hevc",
}
}
pub fn check_available_hwcodec() { pub fn check_available_hwcodec() {
let ctx = EncodeContext { let ctx = EncodeContext {
name: String::from(""), name: String::from(""),
mc_name: None,
width: 1280, width: 1280,
height: 720, height: 720,
pixfmt: DEFAULT_PIXFMT, pixfmt: DEFAULT_PIXFMT,
align: HW_STRIDE_ALIGN as _, align: HW_STRIDE_ALIGN as _,
bitrate: 0, kbs: 0,
timebase: DEFAULT_TIME_BASE, timebase: DEFAULT_TIME_BASE,
gop: DEFAULT_GOP, gop: DEFAULT_GOP,
quality: DEFAULT_HW_QUALITY, quality: DEFAULT_HW_QUALITY,

7
libs/scrap/src/common/vpxcodec.rs
View File

@ -235,6 +235,13 @@ impl EncoderApi for VpxEncoder {
fn support_abr(&self) -> bool { fn support_abr(&self) -> bool {
true true
} }
fn support_changing_quality(&self) -> bool {
true
}
fn latency_free(&self) -> bool {
true
}
} }
impl VpxEncoder { impl VpxEncoder {

8
libs/scrap/src/common/vram.rs
View File

@ -180,6 +180,14 @@ impl EncoderApi for VRamEncoder {
fn support_abr(&self) -> bool { fn support_abr(&self) -> bool {
self.config.device.vendor_id != ADAPTER_VENDOR_INTEL as u32 self.config.device.vendor_id != ADAPTER_VENDOR_INTEL as u32
} }
fn support_changing_quality(&self) -> bool {
true
}
fn latency_free(&self) -> bool {
true
}
} }
impl VRamEncoder { impl VRamEncoder {

7
src/server/video_qos.rs
View File

@ -124,6 +124,10 @@ impl VideoQoS {
self.support_abr.insert(display_idx, support); self.support_abr.insert(display_idx, support);
} }
pub fn in_vbr_state(&self) -> bool {
Config::get_option("enable-abr") != "N" && self.support_abr.iter().all(|e| *e.1)
}
pub fn refresh(&mut self, typ: Option<RefreshType>) { pub fn refresh(&mut self, typ: Option<RefreshType>) {
// fps // fps
let user_fps = |u: &UserData| { let user_fps = |u: &UserData| {
@ -178,8 +182,7 @@ impl VideoQoS {
let mut quality = latest_quality; let mut quality = latest_quality;
// network delay // network delay
let abr_enabled = let abr_enabled = self.in_vbr_state();
Config::get_option("enable-abr") != "N" && self.support_abr.iter().all(|e| *e.1);
if abr_enabled && typ != Some(RefreshType::SetImageQuality) { if abr_enabled && typ != Some(RefreshType::SetImageQuality) {
// max delay // max delay
let delay = self let delay = self

40
src/server/video_service.rs
View File

@ -53,7 +53,7 @@ use scrap::{
codec::{Encoder, EncoderCfg, Quality}, codec::{Encoder, EncoderCfg, Quality},
record::{Recorder, RecorderContext}, record::{Recorder, RecorderContext},
vpxcodec::{VpxEncoderConfig, VpxVideoCodecId}, vpxcodec::{VpxEncoderConfig, VpxVideoCodecId},
CodecFormat, Display, Frame, TraitCapturer, CodecFormat, Display, EncodeInput, Frame, TraitCapturer,
}; };
#[cfg(windows)] #[cfg(windows)]
use std::sync::Once; use std::sync::Once;
@ -470,6 +470,8 @@ fn run(vs: VideoService) -> ResultType<()> {
let mut would_block_count = 0u32; let mut would_block_count = 0u32;
let mut yuv = Vec::new(); let mut yuv = Vec::new();
let mut mid_data = Vec::new(); let mut mid_data = Vec::new();
let mut repeat_encode_counter = 0;
let repeat_encode_max = 10;
while sp.ok() { while sp.ok() {
#[cfg(windows)] #[cfg(windows)]
@ -480,8 +482,15 @@ fn run(vs: VideoService) -> ResultType<()> {
if quality != video_qos.quality() { if quality != video_qos.quality() {
log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality()); log::debug!("quality: {:?} -> {:?}", quality, video_qos.quality());
quality = video_qos.quality(); quality = video_qos.quality();
if encoder.support_changing_quality() {
allow_err!(encoder.set_quality(quality)); allow_err!(encoder.set_quality(quality));
video_qos.store_bitrate(encoder.bitrate()); video_qos.store_bitrate(encoder.bitrate());
} else {
if !video_qos.in_vbr_state() && !quality.is_custom() {
log::info!("switch to change quality");
bail!("SWITCH");
}
}
} }
if client_record != video_qos.record() { if client_record != video_qos.record() {
bail!("SWITCH"); bail!("SWITCH");
@ -526,17 +535,17 @@ fn run(vs: VideoService) -> ResultType<()> {
frame_controller.reset(); frame_controller.reset();
let res = match c.frame(spf) {
Ok(frame) => {
let time = now - start; let time = now - start;
let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64; let ms = (time.as_secs() * 1000 + time.subsec_millis() as u64) as i64;
let res = match c.frame(spf) {
Ok(frame) => {
repeat_encode_counter = 0;
if frame.valid() { if frame.valid() {
let frame = frame.to(encoder.yuvfmt(), &mut yuv, &mut mid_data)?;
let send_conn_ids = handle_one_frame( let send_conn_ids = handle_one_frame(
display_idx, display_idx,
&sp, &sp,
frame, frame,
&mut yuv,
&mut mid_data,
ms, ms,
&mut encoder, &mut encoder,
recorder.clone(), recorder.clone(),
@ -579,6 +588,21 @@ fn run(vs: VideoService) -> ResultType<()> {
} }
} }
} }
if !encoder.latency_free() && yuv.len() > 0 {
// yun.len() > 0 means the frame is not texture.
if repeat_encode_counter < repeat_encode_max {
repeat_encode_counter += 1;
let send_conn_ids = handle_one_frame(
display_idx,
&sp,
EncodeInput::YUV(&yuv),
ms,
&mut encoder,
recorder.clone(),
)?;
frame_controller.set_send(now, send_conn_ids);
}
}
} }
Err(err) => { Err(err) => {
// Get display information again immediately after error. // Get display information again immediately after error.
@ -707,6 +731,7 @@ fn get_encoder_config(
if let Some(hw) = HwRamEncoder::try_get(negotiated_codec) { if let Some(hw) = HwRamEncoder::try_get(negotiated_codec) {
return EncoderCfg::HWRAM(HwRamEncoderConfig { return EncoderCfg::HWRAM(HwRamEncoderConfig {
name: hw.name, name: hw.name,
mc_name: hw.mc_name,
width: c.width, width: c.width,
height: c.height, height: c.height,
quality, quality,
@ -805,9 +830,7 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu
fn handle_one_frame( fn handle_one_frame(
display: usize, display: usize,
sp: &GenericService, sp: &GenericService,
frame: Frame, frame: EncodeInput,
yuv: &mut Vec<u8>,
mid_data: &mut Vec<u8>,
ms: i64, ms: i64,
encoder: &mut Encoder, encoder: &mut Encoder,
recorder: Arc<Mutex<Option<Recorder>>>, recorder: Arc<Mutex<Option<Recorder>>>,
@ -820,7 +843,6 @@ fn handle_one_frame(
Ok(()) Ok(())
})?; })?;
let frame = frame.to(encoder.yuvfmt(), yuv, mid_data)?;
let mut send_conn_ids: HashSet<i32> = Default::default(); let mut send_conn_ids: HashSet<i32> = Default::default();
match encoder.encode_to_message(frame, ms) { match encoder.encode_to_message(frame, ms) {
Ok(mut vf) => { Ok(mut vf) => {