diff --git a/Cargo.lock b/Cargo.lock index e4b5d2792..c4941a1b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3007,6 +3007,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ndk" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0" +dependencies = [ + "bitflags", + "jni-sys", + "ndk-sys 0.4.0", + "num_enum", + "raw-window-handle 0.5.0", + "thiserror", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -3071,6 +3085,15 @@ dependencies = [ "jni-sys", ] +[[package]] +name = "ndk-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21d83ec9c63ec5bf950200a8e508bdad6659972187b625469f58ef8c08e29046" +dependencies = [ + "jni-sys", +] + [[package]] name = "net2" version = "0.2.37" @@ -3948,6 +3971,15 @@ dependencies = [ "cty", ] +[[package]] +name = "raw-window-handle" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7e3d950b66e19e0c372f3fa3fbbcf85b1746b571f74e0c2af6042a5c93420a" +dependencies = [ + "cty", +] + [[package]] name = "rayon" version = "1.5.3" @@ -4412,6 +4444,7 @@ dependencies = [ "lazy_static", "libc", "log", + "ndk 0.7.0", "num_cpus", "quest", "repng", @@ -5857,7 +5890,7 @@ dependencies = [ "objc", "parking_lot 0.11.2", "percent-encoding", - "raw-window-handle", + "raw-window-handle 0.4.3", "smithay-client-toolkit", "wasm-bindgen", "wayland-client", diff --git a/Cargo.toml b/Cargo.toml index f3a0377d4..c65e73d82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ use_dasp = ["dasp"] flutter = ["flutter_rust_bridge"] default = ["use_dasp"] hwcodec = ["scrap/hwcodec"] +mediacodec = ["scrap/mediacodec"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/flutter/android/app/build.gradle b/flutter/android/app/build.gradle index a2a1a02a3..94fc645af 100644 --- a/flutter/android/app/build.gradle +++ b/flutter/android/app/build.gradle @@ -32,7 +32,7 @@ apply plugin: 'kotlin-android' apply from: "$flutterRoot/packages/flutter_tools/gradle/flutter.gradle" android { - compileSdkVersion 32 + compileSdkVersion 33 sourceSets { main.java.srcDirs += 'src/main/kotlin' } diff --git a/flutter/lib/common.dart b/flutter/lib/common.dart index aed6d6fd8..f7c423dbc 100644 --- a/flutter/lib/common.dart +++ b/flutter/lib/common.dart @@ -28,7 +28,7 @@ var isWeb = false; var isWebDesktop = false; var version = ""; int androidVersion = 0; -late final DesktopType? desktopType; +DesktopType? desktopType; typedef F = String Function(String); typedef FMethod = String Function(String, dynamic); diff --git a/flutter/lib/main.dart b/flutter/lib/main.dart index 98ac20bfe..d5837c09c 100644 --- a/flutter/lib/main.dart +++ b/flutter/lib/main.dart @@ -271,7 +271,7 @@ _keepScaleBuilder() { } _registerEventHandler() { - if (desktopType != DesktopType.main) { + if (isDesktop && desktopType != DesktopType.main) { platformFFI.registerEventHandler('theme', 'theme', (evt) async { String? dark = evt['dark']; if (dark != null) { diff --git a/libs/scrap/Cargo.toml b/libs/scrap/Cargo.toml index d40eb0cfd..c980d9d49 100644 --- a/libs/scrap/Cargo.toml +++ b/libs/scrap/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" [features] wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing"] +mediacodec = ["ndk"] [dependencies] block = "0.1" @@ -31,6 +32,7 @@ jni = "0.19" lazy_static = "1.4" log = "0.4" serde_json = "1.0" +ndk = { version = "0.7", features = ["media"], optional = true} [target.'cfg(not(target_os = "android"))'.dev-dependencies] repng = "0.2" diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index f0bd1c5f7..d729342d6 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -7,6 +7,10 @@ use std::{ #[cfg(feature = "hwcodec")] use crate::hwcodec::*; +#[cfg(feature = "mediacodec")] +use crate::mediacodec::{ + MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT, +}; use crate::vpxcodec::*; use hbb_common::{ @@ -15,7 +19,7 @@ use hbb_common::{ message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState}, ResultType, }; -#[cfg(feature = "hwcodec")] +#[cfg(any(feature = "hwcodec", feature = "mediacodec"))] use hbb_common::{ config::{Config2, PeerConfig}, lazy_static, @@ -82,6 +86,8 @@ pub struct Decoder { hw: HwDecoders, #[cfg(feature = "hwcodec")] i420: Vec, + #[cfg(feature = "mediacodec")] + media_codec: MediaCodecDecoders, } #[derive(Debug, Clone)] @@ -242,20 +248,34 @@ impl Decoder { #[cfg(feature = "hwcodec")] if check_hwcodec_config() { let best = HwDecoder::best(); - VideoCodecState { + return VideoCodecState { score_vpx: SCORE_VPX, score_h264: best.h264.map_or(0, |c| c.score), score_h265: best.h265.map_or(0, |c| c.score), perfer: Self::codec_preference(_id).into(), ..Default::default() - } - } else { + }; + } + #[cfg(feature = "mediacodec")] + if check_hwcodec_config() { + let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 92 + } else { + 0 + }; + let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 94 + } else { + 0 + }; return VideoCodecState { score_vpx: SCORE_VPX, + score_h264, + score_h265, + perfer: Self::codec_preference(_id).into(), ..Default::default() }; } - #[cfg(not(feature = "hwcodec"))] VideoCodecState { score_vpx: SCORE_VPX, ..Default::default() @@ -270,6 +290,8 @@ impl Decoder { hw: HwDecoder::new_decoders(), #[cfg(feature = "hwcodec")] i420: vec![], + #[cfg(feature = "mediacodec")] + media_codec: MediaCodecDecoder::new_decoders(), } } @@ -298,6 +320,22 @@ impl Decoder { Err(anyhow!("don't support h265!")) } } + #[cfg(feature = "mediacodec")] + video_frame::Union::H264s(h264s) => { + if let Some(decoder) = &mut self.media_codec.h264 { + Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb) + } else { + Err(anyhow!("don't support h264!")) + } + } + #[cfg(feature = "mediacodec")] + video_frame::Union::H265s(h265s) => { + if let Some(decoder) = &mut self.media_codec.h265 { + Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb) + } else { + Err(anyhow!("don't support h265!")) + } + } _ => Err(anyhow!("unsupported video frame type!")), } } @@ -345,7 +383,20 @@ impl Decoder { return Ok(ret); } - #[cfg(feature = "hwcodec")] + #[cfg(feature = "mediacodec")] + fn handle_mediacodec_video_frame( + decoder: &mut MediaCodecDecoder, + frames: &EncodedVideoFrames, + rgb: &mut Vec, + ) -> ResultType { + let mut ret = false; + for h264 in frames.frames.iter() { + return decoder.decode(&h264.data, rgb); + } + return Ok(false); + } + + #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] fn codec_preference(id: &str) -> PerferCodec { let codec = PeerConfig::load(id) .options @@ -363,7 +414,7 @@ impl Decoder { } } -#[cfg(feature = "hwcodec")] +#[cfg(any(feature = "hwcodec", feature = "mediacodec"))] fn check_hwcodec_config() -> bool { if let Some(v) = Config2::get().options.get("enable-hwcodec") { return v != "N"; diff --git a/libs/scrap/src/common/mediacodec.rs b/libs/scrap/src/common/mediacodec.rs new file mode 100644 index 000000000..fa821246c --- /dev/null +++ b/libs/scrap/src/common/mediacodec.rs @@ -0,0 +1,147 @@ +use hbb_common::anyhow::Error; +use hbb_common::{bail, ResultType}; +use ndk::media::media_codec::{MediaCodec, MediaCodecDirection, MediaFormat}; +use std::ops::Deref; +use std::{ + io::Write, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; + +use crate::{ + codec::{EncoderApi, EncoderCfg}, + I420ToARGB, +}; + +/// MediaCodec mime type name +const H264_MIME_TYPE: &str = "video/avc"; +const H265_MIME_TYPE: &str = "video/hevc"; +// const VP8_MIME_TYPE: &str = "video/x-vnd.on2.vp8"; +// const VP9_MIME_TYPE: &str = "video/x-vnd.on2.vp9"; + +// TODO MediaCodecEncoder + +pub static H264_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false); +pub static H265_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false); + +pub struct MediaCodecDecoder { + decoder: MediaCodec, + name: String, +} + +impl Deref for MediaCodecDecoder { + type Target = MediaCodec; + + fn deref(&self) -> &Self::Target { + &self.decoder + } +} + +pub struct MediaCodecDecoders { + pub h264: Option, + pub h265: Option, +} + +impl MediaCodecDecoder { + pub fn new_decoders() -> MediaCodecDecoders { + let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder); + let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder); + MediaCodecDecoders { h264, h265 } + } + + pub fn decode(&mut self, data: &[u8], rgb: &mut Vec) -> ResultType { + match self.dequeue_input_buffer(Duration::from_millis(10))? { + Some(mut input_buffer) => { + let mut buf = input_buffer.buffer_mut(); + if data.len() > buf.len() { + log::error!("Failed to decode, the input data size is bigger than input buf"); + bail!("The input data size is bigger than input buf"); + } + buf.write_all(&data)?; + self.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)?; + } + None => { + log::debug!("Failed to dequeue_input_buffer: No available input_buffer"); + } + }; + + return match self.dequeue_output_buffer(Duration::from_millis(100))? { + Some(output_buffer) => { + let res_format = self.output_format(); + let w = res_format + .i32("width") + .ok_or(Error::msg("Failed to dequeue_output_buffer, width is None"))? + as usize; + let h = res_format.i32("height").ok_or(Error::msg( + "Failed to dequeue_output_buffer, height is None", + ))? as usize; + let stride = res_format.i32("stride").ok_or(Error::msg( + "Failed to dequeue_output_buffer, stride is None", + ))?; + let buf = output_buffer.buffer(); + let bps = 4; + let u = buf.len() * 2 / 3; + let v = buf.len() * 5 / 6; + rgb.resize(h * w * bps, 0); + let y_ptr = buf.as_ptr(); + let u_ptr = buf[u..].as_ptr(); + let v_ptr = buf[v..].as_ptr(); + unsafe { + I420ToARGB( + y_ptr, + stride, + u_ptr, + stride / 2, + v_ptr, + stride / 2, + rgb.as_mut_ptr(), + (w * bps) as _, + w as _, + h as _, + ); + } + self.release_output_buffer(output_buffer, false)?; + Ok(true) + } + None => { + log::debug!("Failed to dequeue_output: No available dequeue_output"); + Ok(false) + } + }; + } +} + +fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option { + let codec = MediaCodec::from_decoder_type(name)?; + let media_format = MediaFormat::new(); + media_format.set_str("mime", name); + media_format.set_i32("width", 0); + media_format.set_i32("height", 0); + media_format.set_i32("color-format", 19); // COLOR_FormatYUV420Planar + if let Err(e) = codec.configure(&media_format, None, direction) { + log::error!("Failed to init decoder:{:?}", e); + return None; + }; + log::error!("decoder init success"); + if let Err(e) = codec.start() { + log::error!("Failed to start decoder:{:?}", e); + return None; + }; + log::debug!("Init decoder successed!: {:?}", name); + return Some(MediaCodecDecoder { + decoder: codec, + name: name.to_owned(), + }); +} + +pub fn check_mediacodec() { + std::thread::spawn(move || { + // check decoders + let decoders = MediaCodecDecoder::new_decoders(); + H264_DECODER_SUPPORT.swap(decoders.h264.is_some(), Ordering::SeqCst); + H265_DECODER_SUPPORT.swap(decoders.h265.is_some(), Ordering::SeqCst); + decoders.h264.map(|d| d.stop()); + decoders.h265.map(|d| d.stop()); + // TODO encoders + }); +} diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 8ee22ada6..78ea7c888 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -32,6 +32,8 @@ pub mod codec; mod convert; #[cfg(feature = "hwcodec")] pub mod hwcodec; +#[cfg(feature = "mediacodec")] +pub mod mediacodec; pub mod vpxcodec; pub use self::convert::*; pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller diff --git a/src/client.rs b/src/client.rs index cc1f78b78..baf06833a 100644 --- a/src/client.rs +++ b/src/client.rs @@ -1415,11 +1415,9 @@ where let latency_controller = LatencyController::new(); let latency_controller_cl = latency_controller.clone(); - // Create video_handler out of the thread below to ensure that the handler exists before client start. - // It will take a few tenths of a second for the first time, and then tens of milliseconds. - let mut video_handler = VideoHandler::new(latency_controller); std::thread::spawn(move || { + let mut video_handler = VideoHandler::new(latency_controller); loop { if let Ok(data) = video_receiver.recv() { match data { diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index 4fce42da4..e1a806bf4 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -41,6 +41,8 @@ fn initialize(app_dir: &str) { .with_min_level(log::Level::Debug) // limit log level .with_tag("ffi"), // logs will show under mytag tag ); + #[cfg(feature = "mediacodec")] + scrap::mediacodec::check_mediacodec(); } #[cfg(target_os = "ios")] {