diff --git a/Cargo.lock b/Cargo.lock index e4b5d2792..c4941a1b7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3007,6 +3007,20 @@ dependencies = [ "thiserror", ] +[[package]] +name = "ndk" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0" +dependencies = [ + "bitflags", + "jni-sys", + "ndk-sys 0.4.0", + "num_enum", + "raw-window-handle 0.5.0", + "thiserror", +] + [[package]] name = "ndk-context" version = "0.1.1" @@ -3071,6 +3085,15 @@ dependencies = [ "jni-sys", ] +[[package]] +name = "ndk-sys" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21d83ec9c63ec5bf950200a8e508bdad6659972187b625469f58ef8c08e29046" +dependencies = [ + "jni-sys", +] + [[package]] name = "net2" version = "0.2.37" @@ -3948,6 +3971,15 @@ dependencies = [ "cty", ] +[[package]] +name = "raw-window-handle" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed7e3d950b66e19e0c372f3fa3fbbcf85b1746b571f74e0c2af6042a5c93420a" +dependencies = [ + "cty", +] + [[package]] name = "rayon" version = "1.5.3" @@ -4412,6 +4444,7 @@ dependencies = [ "lazy_static", "libc", "log", + "ndk 0.7.0", "num_cpus", "quest", "repng", @@ -5857,7 +5890,7 @@ dependencies = [ "objc", "parking_lot 0.11.2", "percent-encoding", - "raw-window-handle", + "raw-window-handle 0.4.3", "smithay-client-toolkit", "wasm-bindgen", "wayland-client", diff --git a/Cargo.toml b/Cargo.toml index f3a0377d4..c65e73d82 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ use_dasp = ["dasp"] flutter = ["flutter_rust_bridge"] default = ["use_dasp"] hwcodec = ["scrap/hwcodec"] +mediacodec = ["scrap/mediacodec"] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/libs/scrap/Cargo.toml b/libs/scrap/Cargo.toml index d40eb0cfd..c980d9d49 100644 --- a/libs/scrap/Cargo.toml +++ b/libs/scrap/Cargo.toml @@ -11,6 +11,7 @@ edition = "2018" [features] wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing"] +mediacodec = ["ndk"] [dependencies] block = "0.1" @@ -31,6 +32,7 @@ jni = "0.19" lazy_static = "1.4" log = "0.4" serde_json = "1.0" +ndk = { version = "0.7", features = ["media"], optional = true} [target.'cfg(not(target_os = "android"))'.dev-dependencies] repng = "0.2" diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index f0bd1c5f7..d729342d6 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -7,6 +7,10 @@ use std::{ #[cfg(feature = "hwcodec")] use crate::hwcodec::*; +#[cfg(feature = "mediacodec")] +use crate::mediacodec::{ + MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT, +}; use crate::vpxcodec::*; use hbb_common::{ @@ -15,7 +19,7 @@ use hbb_common::{ message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState}, ResultType, }; -#[cfg(feature = "hwcodec")] +#[cfg(any(feature = "hwcodec", feature = "mediacodec"))] use hbb_common::{ config::{Config2, PeerConfig}, lazy_static, @@ -82,6 +86,8 @@ pub struct Decoder { hw: HwDecoders, #[cfg(feature = "hwcodec")] i420: Vec, + #[cfg(feature = "mediacodec")] + media_codec: MediaCodecDecoders, } #[derive(Debug, Clone)] @@ -242,20 +248,34 @@ impl Decoder { #[cfg(feature = "hwcodec")] if check_hwcodec_config() { let best = HwDecoder::best(); - VideoCodecState { + return VideoCodecState { score_vpx: SCORE_VPX, score_h264: best.h264.map_or(0, |c| c.score), score_h265: best.h265.map_or(0, |c| c.score), perfer: Self::codec_preference(_id).into(), ..Default::default() - } - } else { + }; + } + #[cfg(feature = "mediacodec")] + if check_hwcodec_config() { + let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 92 + } else { + 0 + }; + let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) { + 94 + } else { + 0 + }; return VideoCodecState { score_vpx: SCORE_VPX, + score_h264, + score_h265, + perfer: Self::codec_preference(_id).into(), ..Default::default() }; } - #[cfg(not(feature = "hwcodec"))] VideoCodecState { score_vpx: SCORE_VPX, ..Default::default() @@ -270,6 +290,8 @@ impl Decoder { hw: HwDecoder::new_decoders(), #[cfg(feature = "hwcodec")] i420: vec![], + #[cfg(feature = "mediacodec")] + media_codec: MediaCodecDecoder::new_decoders(), } } @@ -298,6 +320,22 @@ impl Decoder { Err(anyhow!("don't support h265!")) } } + #[cfg(feature = "mediacodec")] + video_frame::Union::H264s(h264s) => { + if let Some(decoder) = &mut self.media_codec.h264 { + Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb) + } else { + Err(anyhow!("don't support h264!")) + } + } + #[cfg(feature = "mediacodec")] + video_frame::Union::H265s(h265s) => { + if let Some(decoder) = &mut self.media_codec.h265 { + Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb) + } else { + Err(anyhow!("don't support h265!")) + } + } _ => Err(anyhow!("unsupported video frame type!")), } } @@ -345,7 +383,20 @@ impl Decoder { return Ok(ret); } - #[cfg(feature = "hwcodec")] + #[cfg(feature = "mediacodec")] + fn handle_mediacodec_video_frame( + decoder: &mut MediaCodecDecoder, + frames: &EncodedVideoFrames, + rgb: &mut Vec, + ) -> ResultType { + let mut ret = false; + for h264 in frames.frames.iter() { + return decoder.decode(&h264.data, rgb); + } + return Ok(false); + } + + #[cfg(any(feature = "hwcodec", feature = "mediacodec"))] fn codec_preference(id: &str) -> PerferCodec { let codec = PeerConfig::load(id) .options @@ -363,7 +414,7 @@ impl Decoder { } } -#[cfg(feature = "hwcodec")] +#[cfg(any(feature = "hwcodec", feature = "mediacodec"))] fn check_hwcodec_config() -> bool { if let Some(v) = Config2::get().options.get("enable-hwcodec") { return v != "N"; diff --git a/libs/scrap/src/common/mediacodec.rs b/libs/scrap/src/common/mediacodec.rs index 6a0282b49..fa821246c 100644 --- a/libs/scrap/src/common/mediacodec.rs +++ b/libs/scrap/src/common/mediacodec.rs @@ -1,50 +1,40 @@ -use std::{io::Write, time::Duration}; - +use hbb_common::anyhow::Error; use hbb_common::{bail, ResultType}; -#[cfg(target_os = "android")] use ndk::media::media_codec::{MediaCodec, MediaCodecDirection, MediaFormat}; +use std::ops::Deref; +use std::{ + io::Write, + sync::atomic::{AtomicBool, Ordering}, + time::Duration, +}; use crate::{ codec::{EncoderApi, EncoderCfg}, I420ToARGB, }; -pub struct MediaCodecEncoder { - encoder: MediaCodec, -} +/// MediaCodec mime type name +const H264_MIME_TYPE: &str = "video/avc"; +const H265_MIME_TYPE: &str = "video/hevc"; +// const VP8_MIME_TYPE: &str = "video/x-vnd.on2.vp8"; +// const VP9_MIME_TYPE: &str = "video/x-vnd.on2.vp9"; -impl EncoderApi for MediaCodecEncoder { - fn new(cfg: EncoderCfg) -> ResultType - where - Self: Sized, - { - if let EncoderCfg::HW(cfg) = cfg { - create_media_codec(&cfg.codec_name, MediaCodecDirection::Encoder) - } else { - bail!("encoder type mismatch") - } - } +// TODO MediaCodecEncoder - fn encode_to_message( - &mut self, - frame: &[u8], - ms: i64, - ) -> ResultType { - todo!() - } - - fn use_yuv(&self) -> bool { - todo!() - } - - fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> { - todo!() - } -} +pub static H264_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false); +pub static H265_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false); pub struct MediaCodecDecoder { decoder: MediaCodec, - // pub info: CodecInfo, + name: String, +} + +impl Deref for MediaCodecDecoder { + type Target = MediaCodec; + + fn deref(&self) -> &Self::Target { + &self.decoder + } } pub struct MediaCodecDecoders { @@ -52,85 +42,50 @@ pub struct MediaCodecDecoders { pub h265: Option, } -// "video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) -// "video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) -// "video/avc" - H.264/AVC video -// "video/hevc" - H.265/HEVC video - impl MediaCodecDecoder { pub fn new_decoders() -> MediaCodecDecoders { - // 直接生成 h264 和 h265 - // 264 - let h264 = create_media_codec("video/avc", MediaCodecDirection::Decoder) - .map(|decoder| MediaCodecDecoder { decoder }); - let h265 = create_media_codec("video/hevc", MediaCodecDirection::Decoder) - .map(|decoder| MediaCodecDecoder { decoder }); - + let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder); + let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder); MediaCodecDecoders { h264, h265 } } pub fn decode(&mut self, data: &[u8], rgb: &mut Vec) -> ResultType { - log::debug!("start dequeue_input"); - - match self - .decoder - .dequeue_input_buffer(Duration::from_millis(10)) - .unwrap() - { + match self.dequeue_input_buffer(Duration::from_millis(10))? { Some(mut input_buffer) => { let mut buf = input_buffer.buffer_mut(); - log::debug!( - "dequeue_input success:buf ptr:{:?},len:{}", - buf.as_ptr(), - buf.len() - ); if data.len() > buf.len() { - log::error!("break! res.len()>buf.len()"); - bail!("break! res.len()>buf.len()"); + log::error!("Failed to decode, the input data size is bigger than input buf"); + bail!("The input data size is bigger than input buf"); } - buf.write_all(&data).unwrap(); - if let Err(e) = self - .decoder - .queue_input_buffer(input_buffer, 0, data.len(), 0, 0) - { - log::debug!("debug queue_input_buffer:{:?}", e); - }; + buf.write_all(&data)?; + self.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)?; } None => { - log::debug!("dequeue_input_buffer fail :None"); + log::debug!("Failed to dequeue_input_buffer: No available input_buffer"); } }; - return match self - .decoder - .dequeue_output_buffer(Duration::from_millis(100)) - { - Ok(Some(output_buffer)) => { - log::debug!("dequeue_output success"); - // let res_format = output_buffer.format(); - let res_format = self.decoder.output_format(); - log::debug!("res_format:{:?}", res_format.str("mime")); - log::debug!("res_color:{:?}", res_format.i32("color-format")); - log::debug!("stride:{:?}", res_format.i32("stride")); - let w = res_format.i32("width").unwrap() as usize; - let h = res_format.i32("height").unwrap() as usize; - let stride = res_format.i32("stride").unwrap(); // todo - - // let w = 1920; - // let h = 1080; - // let stride = 1920; // todo - + return match self.dequeue_output_buffer(Duration::from_millis(100))? { + Some(output_buffer) => { + let res_format = self.output_format(); + let w = res_format + .i32("width") + .ok_or(Error::msg("Failed to dequeue_output_buffer, width is None"))? + as usize; + let h = res_format.i32("height").ok_or(Error::msg( + "Failed to dequeue_output_buffer, height is None", + ))? as usize; + let stride = res_format.i32("stride").ok_or(Error::msg( + "Failed to dequeue_output_buffer, stride is None", + ))?; let buf = output_buffer.buffer(); - log::debug!("output_buffer ptr:{:?} len:{}", buf.as_ptr(), buf.len()); let bps = 4; let u = buf.len() * 2 / 3; let v = buf.len() * 5 / 6; rgb.resize(h * w * bps, 0); - log::debug!("start I420ToARGB,u:{},v:{},w:{},h:{}", u, v, w, h); let y_ptr = buf.as_ptr(); let u_ptr = buf[u..].as_ptr(); let v_ptr = buf[v..].as_ptr(); - log::debug!("ptr,y:{:?},u:{:?},v:{:?}", y_ptr, u_ptr, v_ptr); unsafe { I420ToARGB( y_ptr, @@ -145,43 +100,48 @@ impl MediaCodecDecoder { h as _, ); } - log::debug!("end I420ToARGB"); - log::debug!("release_output_buffer"); - self.decoder - .release_output_buffer(output_buffer, false) - .unwrap(); - log::debug!("return true"); + self.release_output_buffer(output_buffer, false)?; Ok(true) } - Ok(None) => { - log::debug!("dequeue_output fail :None"); - Ok(false) - } - Err(e) => { - log::debug!("dequeue_output fail :error:{:?}", e); + None => { + log::debug!("Failed to dequeue_output: No available dequeue_output"); Ok(false) } }; } } -fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option { - let codec = MediaCodec::from_decoder_type(name).unwrap(); - log::debug!("start init"); +fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option { + let codec = MediaCodec::from_decoder_type(name)?; let media_format = MediaFormat::new(); media_format.set_str("mime", name); media_format.set_i32("width", 0); media_format.set_i32("height", 0); media_format.set_i32("color-format", 19); // COLOR_FormatYUV420Planar if let Err(e) = codec.configure(&media_format, None, direction) { - log::error!("failed to decoder.init:{:?}", e); + log::error!("Failed to init decoder:{:?}", e); return None; }; log::error!("decoder init success"); if let Err(e) = codec.start() { - log::error!("failed to decoder.start:{:?}", e); + log::error!("Failed to start decoder:{:?}", e); return None; }; - log::debug!("init decoder successed!:{:?}", name); - return Some(codec); + log::debug!("Init decoder successed!: {:?}", name); + return Some(MediaCodecDecoder { + decoder: codec, + name: name.to_owned(), + }); +} + +pub fn check_mediacodec() { + std::thread::spawn(move || { + // check decoders + let decoders = MediaCodecDecoder::new_decoders(); + H264_DECODER_SUPPORT.swap(decoders.h264.is_some(), Ordering::SeqCst); + H265_DECODER_SUPPORT.swap(decoders.h265.is_some(), Ordering::SeqCst); + decoders.h264.map(|d| d.stop()); + decoders.h265.map(|d| d.stop()); + // TODO encoders + }); } diff --git a/libs/scrap/src/common/mod.rs b/libs/scrap/src/common/mod.rs index 8ee22ada6..78ea7c888 100644 --- a/libs/scrap/src/common/mod.rs +++ b/libs/scrap/src/common/mod.rs @@ -32,6 +32,8 @@ pub mod codec; mod convert; #[cfg(feature = "hwcodec")] pub mod hwcodec; +#[cfg(feature = "mediacodec")] +pub mod mediacodec; pub mod vpxcodec; pub use self::convert::*; pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller diff --git a/src/flutter_ffi.rs b/src/flutter_ffi.rs index c3b71e4c9..bc082aedf 100644 --- a/src/flutter_ffi.rs +++ b/src/flutter_ffi.rs @@ -49,6 +49,8 @@ fn initialize(app_dir: &str) { .with_min_level(log::Level::Debug) // limit log level .with_tag("ffi"), // logs will show under mytag tag ); + #[cfg(feature = "mediacodec")] + scrap::mediacodec::check_mediacodec(); } #[cfg(target_os = "ios")] {