hongshaorou/rustdesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1542 from Heap-Hop/ndk_mediacodec

Browse Source 
Feat: Android H264/H265 decoder support
This commit is contained in:
RustDesk 2022-09-15 21:17:24 +08:00 committed by GitHub
commit 5f6619080f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 250 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
Cargo.lock generated
View File

@ -3007,6 +3007,20 @@ dependencies = [
"thiserror",
]
[[package]]
name = "ndk"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0"
dependencies = [
"bitflags",
"jni-sys",
"ndk-sys 0.4.0",
"num_enum",
"raw-window-handle 0.5.0",
"thiserror",
]
[[package]]
name = "ndk-context"
version = "0.1.1"
@ -3071,6 +3085,15 @@ dependencies = [
"jni-sys",
]
[[package]]
name = "ndk-sys"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21d83ec9c63ec5bf950200a8e508bdad6659972187b625469f58ef8c08e29046"
dependencies = [
"jni-sys",
]
[[package]]
name = "net2"
version = "0.2.37"
@ -3948,6 +3971,15 @@ dependencies = [
"cty",
]
[[package]]
name = "raw-window-handle"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed7e3d950b66e19e0c372f3fa3fbbcf85b1746b571f74e0c2af6042a5c93420a"
dependencies = [
"cty",
]
[[package]]
name = "rayon"
version = "1.5.3"
@ -4412,6 +4444,7 @@ dependencies = [
"lazy_static",
"libc",
"log",
"ndk 0.7.0",
"num_cpus",
"quest",
"repng",
@ -5857,7 +5890,7 @@ dependencies = [
"objc",
"parking_lot 0.11.2",
"percent-encoding",
"raw-window-handle",
"raw-window-handle 0.4.3",
"smithay-client-toolkit",
"wasm-bindgen",
"wayland-client",

1
Cargo.toml
View File

@ -28,6 +28,7 @@ use_dasp = ["dasp"]
flutter = ["flutter_rust_bridge"]
default = ["use_dasp"]
hwcodec = ["scrap/hwcodec"]
mediacodec = ["scrap/mediacodec"]
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

2
flutter/android/app/build.gradle
View File

@ -32,7 +32,7 @@ apply plugin: 'kotlin-android'
apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle"
android {
compileSdkVersion 32
compileSdkVersion 33
sourceSets {
main.java.srcDirs += 'src/main/kotlin'
}

2
flutter/lib/common.dart
View File

@ -28,7 +28,7 @@ var isWeb = false;
var isWebDesktop = false;
var version = "";
int androidVersion = 0;
late final DesktopType? desktopType;
DesktopType? desktopType;
typedef F = String Function(String);
typedef FMethod = String Function(String, dynamic);

2
flutter/lib/main.dart
View File

@ -271,7 +271,7 @@ _keepScaleBuilder() {
}
_registerEventHandler() {
if (desktopType != DesktopType.main) {
if (isDesktop && desktopType != DesktopType.main) {
platformFFI.registerEventHandler('theme', 'theme', (evt) async {
String? dark = evt['dark'];
if (dark != null) {

2
libs/scrap/Cargo.toml
View File

@ -11,6 +11,7 @@ edition = "2018"
[features]
wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing"]
mediacodec = ["ndk"]
[dependencies]
block = "0.1"
@ -31,6 +32,7 @@ jni = "0.19"
lazy_static = "1.4"
log = "0.4"
serde_json = "1.0"
ndk = { version = "0.7", features = ["media"], optional = true}
[target.'cfg(not(target_os = "android"))'.dev-dependencies]
repng = "0.2"

65
libs/scrap/src/common/codec.rs
View File

@ -7,6 +7,10 @@ use std::{
#[cfg(feature = "hwcodec")]
use crate::hwcodec::*;
#[cfg(feature = "mediacodec")]
use crate::mediacodec::{
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
};
use crate::vpxcodec::*;
use hbb_common::{
@ -15,7 +19,7 @@ use hbb_common::{
message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState},
ResultType,
};
#[cfg(feature = "hwcodec")]
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
use hbb_common::{
config::{Config2, PeerConfig},
lazy_static,
@ -82,6 +86,8 @@ pub struct Decoder {
hw: HwDecoders,
#[cfg(feature = "hwcodec")]
i420: Vec<u8>,
#[cfg(feature = "mediacodec")]
media_codec: MediaCodecDecoders,
}
#[derive(Debug, Clone)]
@ -242,20 +248,34 @@ impl Decoder {
#[cfg(feature = "hwcodec")]
if check_hwcodec_config() {
let best = HwDecoder::best();
VideoCodecState {
return VideoCodecState {
score_vpx: SCORE_VPX,
score_h264: best.h264.map_or(0, |c| c.score),
score_h265: best.h265.map_or(0, |c| c.score),
perfer: Self::codec_preference(_id).into(),
..Default::default()
}
} else {
};
}
#[cfg(feature = "mediacodec")]
if check_hwcodec_config() {
let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
92
} else {
0
};
let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
94
} else {
0
};
return VideoCodecState {
score_vpx: SCORE_VPX,
score_h264,
score_h265,
perfer: Self::codec_preference(_id).into(),
..Default::default()
};
}
#[cfg(not(feature = "hwcodec"))]
VideoCodecState {
score_vpx: SCORE_VPX,
..Default::default()
@ -270,6 +290,8 @@ impl Decoder {
hw: HwDecoder::new_decoders(),
#[cfg(feature = "hwcodec")]
i420: vec![],
#[cfg(feature = "mediacodec")]
media_codec: MediaCodecDecoder::new_decoders(),
}
}
@ -298,6 +320,22 @@ impl Decoder {
Err(anyhow!("don't support h265!"))
}
}
#[cfg(feature = "mediacodec")]
video_frame::Union::H264s(h264s) => {
if let Some(decoder) = &mut self.media_codec.h264 {
Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
} else {
Err(anyhow!("don't support h264!"))
}
}
#[cfg(feature = "mediacodec")]
video_frame::Union::H265s(h265s) => {
if let Some(decoder) = &mut self.media_codec.h265 {
Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
} else {
Err(anyhow!("don't support h265!"))
}
}
_ => Err(anyhow!("unsupported video frame type!")),
}
}
@ -345,7 +383,20 @@ impl Decoder {
return Ok(ret);
}
#[cfg(feature = "hwcodec")]
#[cfg(feature = "mediacodec")]
fn handle_mediacodec_video_frame(
decoder: &mut MediaCodecDecoder,
frames: &EncodedVideoFrames,
rgb: &mut Vec<u8>,
) -> ResultType<bool> {
let mut ret = false;
for h264 in frames.frames.iter() {
return decoder.decode(&h264.data, rgb);
}
return Ok(false);
}
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
fn codec_preference(id: &str) -> PerferCodec {
let codec = PeerConfig::load(id)
.options
@ -363,7 +414,7 @@ impl Decoder {
}
}
#[cfg(feature = "hwcodec")]
#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
fn check_hwcodec_config() -> bool {
if let Some(v) = Config2::get().options.get("enable-hwcodec") {
return v != "N";

147
libs/scrap/src/common/mediacodec.rs Normal file
View File

@ -0,0 +1,147 @@
use hbb_common::anyhow::Error;
use hbb_common::{bail, ResultType};
use ndk::media::media_codec::{MediaCodec, MediaCodecDirection, MediaFormat};
use std::ops::Deref;
use std::{
io::Write,
sync::atomic::{AtomicBool, Ordering},
time::Duration,
};
use crate::{
codec::{EncoderApi, EncoderCfg},
I420ToARGB,
};
/// MediaCodec mime type name
const H264_MIME_TYPE: &str = "video/avc";
const H265_MIME_TYPE: &str = "video/hevc";
// const VP8_MIME_TYPE: &str = "video/x-vnd.on2.vp8";
// const VP9_MIME_TYPE: &str = "video/x-vnd.on2.vp9";
// TODO MediaCodecEncoder
pub static H264_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
pub static H265_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
pub struct MediaCodecDecoder {
decoder: MediaCodec,
name: String,
}
impl Deref for MediaCodecDecoder {
type Target = MediaCodec;
fn deref(&self) -> &Self::Target {
&self.decoder
}
}
pub struct MediaCodecDecoders {
pub h264: Option<MediaCodecDecoder>,
pub h265: Option<MediaCodecDecoder>,
}
impl MediaCodecDecoder {
pub fn new_decoders() -> MediaCodecDecoders {
let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder);
let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder);
MediaCodecDecoders { h264, h265 }
}
pub fn decode(&mut self, data: &[u8], rgb: &mut Vec<u8>) -> ResultType<bool> {
match self.dequeue_input_buffer(Duration::from_millis(10))? {
Some(mut input_buffer) => {
let mut buf = input_buffer.buffer_mut();
if data.len() > buf.len() {
log::error!("Failed to decode, the input data size is bigger than input buf");
bail!("The input data size is bigger than input buf");
}
buf.write_all(&data)?;
self.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)?;
}
None => {
log::debug!("Failed to dequeue_input_buffer: No available input_buffer");
}
};
return match self.dequeue_output_buffer(Duration::from_millis(100))? {
Some(output_buffer) => {
let res_format = self.output_format();
let w = res_format
.i32("width")
.ok_or(Error::msg("Failed to dequeue_output_buffer, width is None"))?
as usize;
let h = res_format.i32("height").ok_or(Error::msg(
"Failed to dequeue_output_buffer, height is None",
))? as usize;
let stride = res_format.i32("stride").ok_or(Error::msg(
"Failed to dequeue_output_buffer, stride is None",
))?;
let buf = output_buffer.buffer();
let bps = 4;
let u = buf.len() * 2 / 3;
let v = buf.len() * 5 / 6;
rgb.resize(h * w * bps, 0);
let y_ptr = buf.as_ptr();
let u_ptr = buf[u..].as_ptr();
let v_ptr = buf[v..].as_ptr();
unsafe {
I420ToARGB(
y_ptr,
stride,
u_ptr,
stride / 2,
v_ptr,
stride / 2,
rgb.as_mut_ptr(),
(w * bps) as _,
w as _,
h as _,
);
}
self.release_output_buffer(output_buffer, false)?;
Ok(true)
}
None => {
log::debug!("Failed to dequeue_output: No available dequeue_output");
Ok(false)
}
};
}
}
fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option<MediaCodecDecoder> {
let codec = MediaCodec::from_decoder_type(name)?;
let media_format = MediaFormat::new();
media_format.set_str("mime", name);
media_format.set_i32("width", 0);
media_format.set_i32("height", 0);
media_format.set_i32("color-format", 19); // COLOR_FormatYUV420Planar
if let Err(e) = codec.configure(&media_format, None, direction) {
log::error!("Failed to init decoder:{:?}", e);
return None;
};
log::error!("decoder init success");
if let Err(e) = codec.start() {
log::error!("Failed to start decoder:{:?}", e);
return None;
};
log::debug!("Init decoder successed!: {:?}", name);
return Some(MediaCodecDecoder {
decoder: codec,
name: name.to_owned(),
});
}
pub fn check_mediacodec() {
std::thread::spawn(move || {
// check decoders
let decoders = MediaCodecDecoder::new_decoders();
H264_DECODER_SUPPORT.swap(decoders.h264.is_some(), Ordering::SeqCst);
H265_DECODER_SUPPORT.swap(decoders.h265.is_some(), Ordering::SeqCst);
decoders.h264.map(|d| d.stop());
decoders.h265.map(|d| d.stop());
// TODO encoders
});
}

2
libs/scrap/src/common/mod.rs
View File

@ -32,6 +32,8 @@ pub mod codec;
mod convert;
#[cfg(feature = "hwcodec")]
pub mod hwcodec;
#[cfg(feature = "mediacodec")]
pub mod mediacodec;
pub mod vpxcodec;
pub use self::convert::*;
pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller

4
src/client.rs
View File

@ -1415,11 +1415,9 @@ where
let latency_controller = LatencyController::new();
let latency_controller_cl = latency_controller.clone();
// Create video_handler out of the thread below to ensure that the handler exists before client start.
// It will take a few tenths of a second for the first time, and then tens of milliseconds.
let mut video_handler = VideoHandler::new(latency_controller);
std::thread::spawn(move || {
let mut video_handler = VideoHandler::new(latency_controller);
loop {
if let Ok(data) = video_receiver.recv() {
match data {

2
src/flutter_ffi.rs
View File

@ -41,6 +41,8 @@ fn initialize(app_dir: &str) {
.with_min_level(log::Level::Debug) // limit log level
.with_tag("ffi"), // logs will show under mytag tag
);
#[cfg(feature = "mediacodec")]
scrap::mediacodec::check_mediacodec();
}
#[cfg(target_os = "ios")]
{