feat mediacodec: Android H264/H265 decoder support

Cargo.lock

@@ -3007,6 +3007,20 @@ dependencies = [
  "thiserror",
 ]
 
+[[package]]
+name = "ndk"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "451422b7e4718271c8b5b3aadf5adedba43dc76312454b387e98fae0fc951aa0"
+dependencies = [
+ "bitflags",
+ "jni-sys",
+ "ndk-sys 0.4.0",
+ "num_enum",
+ "raw-window-handle 0.5.0",
+ "thiserror",
+]
+
 [[package]]
 name = "ndk-context"
 version = "0.1.1"
@@ -3071,6 +3085,15 @@ dependencies = [
  "jni-sys",
 ]
 
+[[package]]
+name = "ndk-sys"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "21d83ec9c63ec5bf950200a8e508bdad6659972187b625469f58ef8c08e29046"
+dependencies = [
+ "jni-sys",
+]
+
 [[package]]
 name = "net2"
 version = "0.2.37"
@@ -3948,6 +3971,15 @@ dependencies = [
  "cty",
 ]
 
+[[package]]
+name = "raw-window-handle"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed7e3d950b66e19e0c372f3fa3fbbcf85b1746b571f74e0c2af6042a5c93420a"
+dependencies = [
+ "cty",
+]
+
 [[package]]
 name = "rayon"
 version = "1.5.3"
@@ -4412,6 +4444,7 @@ dependencies = [
  "lazy_static",
  "libc",
  "log",
+ "ndk 0.7.0",
  "num_cpus",
  "quest",
  "repng",
@@ -5857,7 +5890,7 @@ dependencies = [
  "objc",
  "parking_lot 0.11.2",
  "percent-encoding",
- "raw-window-handle",
+ "raw-window-handle 0.4.3",
  "smithay-client-toolkit",
  "wasm-bindgen",
  "wayland-client",

Cargo.toml

@@ -28,6 +28,7 @@ use_dasp = ["dasp"]
 flutter = ["flutter_rust_bridge"]
 default = ["use_dasp"]
 hwcodec = ["scrap/hwcodec"]
+mediacodec = ["scrap/mediacodec"]
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

libs/scrap/Cargo.toml

@@ -11,6 +11,7 @@ edition = "2018"
 
 [features]
 wayland = ["gstreamer", "gstreamer-app", "gstreamer-video", "dbus", "tracing"]
+mediacodec = ["ndk"]
 
 [dependencies]
 block = "0.1"
@@ -31,6 +32,7 @@ jni = "0.19"
 lazy_static = "1.4"
 log = "0.4"
 serde_json = "1.0"
+ndk = { version = "0.7", features = ["media"], optional = true}
 
 [target.'cfg(not(target_os = "android"))'.dev-dependencies]
 repng = "0.2"

libs/scrap/src/common/codec.rs

@@ -7,6 +7,10 @@ use std::{
 
 #[cfg(feature = "hwcodec")]
 use crate::hwcodec::*;
+#[cfg(feature = "mediacodec")]
+use crate::mediacodec::{
+    MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
+};
 use crate::vpxcodec::*;
 
 use hbb_common::{
@@ -15,7 +19,7 @@ use hbb_common::{
     message_proto::{video_frame, EncodedVideoFrames, Message, VideoCodecState},
     ResultType,
 };
-#[cfg(feature = "hwcodec")]
+#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
 use hbb_common::{
     config::{Config2, PeerConfig},
     lazy_static,
@@ -82,6 +86,8 @@ pub struct Decoder {
     hw: HwDecoders,
     #[cfg(feature = "hwcodec")]
     i420: Vec<u8>,
+    #[cfg(feature = "mediacodec")]
+    media_codec: MediaCodecDecoders,
 }
 
 #[derive(Debug, Clone)]
@@ -242,20 +248,34 @@ impl Decoder {
         #[cfg(feature = "hwcodec")]
         if check_hwcodec_config() {
             let best = HwDecoder::best();
-            VideoCodecState {
+            return VideoCodecState {
                 score_vpx: SCORE_VPX,
                 score_h264: best.h264.map_or(0, |c| c.score),
                 score_h265: best.h265.map_or(0, |c| c.score),
                 perfer: Self::codec_preference(_id).into(),
                 ..Default::default()
-            }
+            };
-        } else {
+        }
+        #[cfg(feature = "mediacodec")]
+        if check_hwcodec_config() {
+            let score_h264 = if H264_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
+                92
+            } else {
+                0
+            };
+            let score_h265 = if H265_DECODER_SUPPORT.load(std::sync::atomic::Ordering::SeqCst) {
+                94
+            } else {
+                0
+            };
             return VideoCodecState {
                 score_vpx: SCORE_VPX,
+                score_h264,
+                score_h265,
+                perfer: Self::codec_preference(_id).into(),
                 ..Default::default()
             };
         }
-        #[cfg(not(feature = "hwcodec"))]
         VideoCodecState {
             score_vpx: SCORE_VPX,
             ..Default::default()
@@ -270,6 +290,8 @@ impl Decoder {
             hw: HwDecoder::new_decoders(),
             #[cfg(feature = "hwcodec")]
             i420: vec![],
+            #[cfg(feature = "mediacodec")]
+            media_codec: MediaCodecDecoder::new_decoders(),
         }
     }
 
@@ -298,6 +320,22 @@ impl Decoder {
                     Err(anyhow!("don't support h265!"))
                 }
             }
+            #[cfg(feature = "mediacodec")]
+            video_frame::Union::H264s(h264s) => {
+                if let Some(decoder) = &mut self.media_codec.h264 {
+                    Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
+                } else {
+                    Err(anyhow!("don't support h264!"))
+                }
+            }
+            #[cfg(feature = "mediacodec")]
+            video_frame::Union::H265s(h265s) => {
+                if let Some(decoder) = &mut self.media_codec.h265 {
+                    Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
+                } else {
+                    Err(anyhow!("don't support h265!"))
+                }
+            }
             _ => Err(anyhow!("unsupported video frame type!")),
         }
     }
@@ -345,7 +383,20 @@ impl Decoder {
         return Ok(ret);
     }
 
-    #[cfg(feature = "hwcodec")]
+    #[cfg(feature = "mediacodec")]
+    fn handle_mediacodec_video_frame(
+        decoder: &mut MediaCodecDecoder,
+        frames: &EncodedVideoFrames,
+        rgb: &mut Vec<u8>,
+    ) -> ResultType<bool> {
+        let mut ret = false;
+        for h264 in frames.frames.iter() {
+            return decoder.decode(&h264.data, rgb);
+        }
+        return Ok(false);
+    }
+
+    #[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
     fn codec_preference(id: &str) -> PerferCodec {
         let codec = PeerConfig::load(id)
             .options
@@ -363,7 +414,7 @@ impl Decoder {
     }
 }
 
-#[cfg(feature = "hwcodec")]
+#[cfg(any(feature = "hwcodec", feature = "mediacodec"))]
 fn check_hwcodec_config() -> bool {
     if let Some(v) = Config2::get().options.get("enable-hwcodec") {
         return v != "N";

libs/scrap/src/common/mediacodec.rs

@@ -1,50 +1,40 @@
-use std::{io::Write, time::Duration};
+use hbb_common::anyhow::Error;
-
 use hbb_common::{bail, ResultType};
-#[cfg(target_os = "android")]
 use ndk::media::media_codec::{MediaCodec, MediaCodecDirection, MediaFormat};
+use std::ops::Deref;
+use std::{
+    io::Write,
+    sync::atomic::{AtomicBool, Ordering},
+    time::Duration,
+};
 
 use crate::{
     codec::{EncoderApi, EncoderCfg},
     I420ToARGB,
 };
 
-pub struct MediaCodecEncoder {
+/// MediaCodec mime type name
-    encoder: MediaCodec,
+const H264_MIME_TYPE: &str = "video/avc";
-}
+const H265_MIME_TYPE: &str = "video/hevc";
+// const VP8_MIME_TYPE: &str = "video/x-vnd.on2.vp8";
+// const VP9_MIME_TYPE: &str = "video/x-vnd.on2.vp9";
 
-impl EncoderApi for MediaCodecEncoder {
+// TODO MediaCodecEncoder
-    fn new(cfg: EncoderCfg) -> ResultType<Self>
-    where
-        Self: Sized,
-    {
-        if let EncoderCfg::HW(cfg) = cfg {
-            create_media_codec(&cfg.codec_name, MediaCodecDirection::Encoder)
-        } else {
-            bail!("encoder type mismatch")
-        }
-    }
 
-    fn encode_to_message(
+pub static H264_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
-        &mut self,
+pub static H265_DECODER_SUPPORT: AtomicBool = AtomicBool::new(false);
-        frame: &[u8],
-        ms: i64,
-    ) -> ResultType<hbb_common::message_proto::Message> {
-        todo!()
-    }
-
-    fn use_yuv(&self) -> bool {
-        todo!()
-    }
-
-    fn set_bitrate(&mut self, bitrate: u32) -> ResultType<()> {
-        todo!()
-    }
-}
 
 pub struct MediaCodecDecoder {
     decoder: MediaCodec,
-    // pub info: CodecInfo,
+    name: String,
+}
+
+impl Deref for MediaCodecDecoder {
+    type Target = MediaCodec;
+
+    fn deref(&self) -> &Self::Target {
+        &self.decoder
+    }
 }
 
 pub struct MediaCodecDecoders {
@@ -52,85 +42,50 @@ pub struct MediaCodecDecoders {
     pub h265: Option<MediaCodecDecoder>,
 }
 
-// "video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm)
-// "video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm)
-// "video/avc" - H.264/AVC video
-// "video/hevc" - H.265/HEVC video
-
 impl MediaCodecDecoder {
     pub fn new_decoders() -> MediaCodecDecoders {
-        // 直接生成 h264 和 h265
+        let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder);
-        // 264
+        let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder);
-        let h264 = create_media_codec("video/avc", MediaCodecDirection::Decoder)
-            .map(|decoder| MediaCodecDecoder { decoder });
-        let h265 = create_media_codec("video/hevc", MediaCodecDirection::Decoder)
-            .map(|decoder| MediaCodecDecoder { decoder });
-
         MediaCodecDecoders { h264, h265 }
     }
 
     pub fn decode(&mut self, data: &[u8], rgb: &mut Vec<u8>) -> ResultType<bool> {
-        log::debug!("start dequeue_input");
+        match self.dequeue_input_buffer(Duration::from_millis(10))? {
-
-        match self
-            .decoder
-            .dequeue_input_buffer(Duration::from_millis(10))
-            .unwrap()
-        {
             Some(mut input_buffer) => {
                 let mut buf = input_buffer.buffer_mut();
-                log::debug!(
-                    "dequeue_input success:buf ptr:{:?},len:{}",
-                    buf.as_ptr(),
-                    buf.len()
-                );
                 if data.len() > buf.len() {
-                    log::error!("break! res.len()>buf.len()");
+                    log::error!("Failed to decode, the input data size is bigger than input buf");
-                    bail!("break! res.len()>buf.len()");
+                    bail!("The input data size is bigger than input buf");
                 }
-                buf.write_all(&data).unwrap();
+                buf.write_all(&data)?;
-                if let Err(e) = self
+                self.queue_input_buffer(input_buffer, 0, data.len(), 0, 0)?;
-                    .decoder
-                    .queue_input_buffer(input_buffer, 0, data.len(), 0, 0)
-                {
-                    log::debug!("debug queue_input_buffer:{:?}", e);
-                };
             }
             None => {
-                log::debug!("dequeue_input_buffer fail :None");
+                log::debug!("Failed to dequeue_input_buffer: No available input_buffer");
             }
         };
 
-        return match self
+        return match self.dequeue_output_buffer(Duration::from_millis(100))? {
-            .decoder
+            Some(output_buffer) => {
-            .dequeue_output_buffer(Duration::from_millis(100))
+                let res_format = self.output_format();
-        {
+                let w = res_format
-            Ok(Some(output_buffer)) => {
+                    .i32("width")
-                log::debug!("dequeue_output success");
+                    .ok_or(Error::msg("Failed to dequeue_output_buffer, width is None"))?
-                // let res_format = output_buffer.format();
+                    as usize;
-                let res_format = self.decoder.output_format();
+                let h = res_format.i32("height").ok_or(Error::msg(
-                log::debug!("res_format:{:?}", res_format.str("mime"));
+                    "Failed to dequeue_output_buffer, height is None",
-                log::debug!("res_color:{:?}", res_format.i32("color-format"));
+                ))? as usize;
-                log::debug!("stride:{:?}", res_format.i32("stride"));
+                let stride = res_format.i32("stride").ok_or(Error::msg(
-                let w = res_format.i32("width").unwrap() as usize;
+                    "Failed to dequeue_output_buffer, stride is None",
-                let h = res_format.i32("height").unwrap() as usize;
+                ))?;
-                let stride = res_format.i32("stride").unwrap(); // todo
-
-                // let w = 1920;
-                // let h = 1080;
-                // let stride = 1920; // todo
-
                 let buf = output_buffer.buffer();
-                log::debug!("output_buffer ptr:{:?} len:{}", buf.as_ptr(), buf.len());
                 let bps = 4;
                 let u = buf.len() * 2 / 3;
                 let v = buf.len() * 5 / 6;
                 rgb.resize(h * w * bps, 0);
-                log::debug!("start I420ToARGB,u:{},v:{},w:{},h:{}", u, v, w, h);
                 let y_ptr = buf.as_ptr();
                 let u_ptr = buf[u..].as_ptr();
                 let v_ptr = buf[v..].as_ptr();
-                log::debug!("ptr,y:{:?},u:{:?},v:{:?}", y_ptr, u_ptr, v_ptr);
                 unsafe {
                     I420ToARGB(
                         y_ptr,
@@ -145,43 +100,48 @@ impl MediaCodecDecoder {
                         h as _,
                     );
                 }
-                log::debug!("end I420ToARGB");
+                self.release_output_buffer(output_buffer, false)?;
-                log::debug!("release_output_buffer");
-                self.decoder
-                    .release_output_buffer(output_buffer, false)
-                    .unwrap();
-                log::debug!("return true");
                 Ok(true)
             }
-            Ok(None) => {
+            None => {
-                log::debug!("dequeue_output fail :None");
+                log::debug!("Failed to dequeue_output: No available dequeue_output");
-                Ok(false)
-            }
-            Err(e) => {
-                log::debug!("dequeue_output fail :error:{:?}", e);
                 Ok(false)
             }
         };
     }
 }
 
-fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option<MediaCodec> {
+fn create_media_codec(name: &str, direction: MediaCodecDirection) -> Option<MediaCodecDecoder> {
-    let codec = MediaCodec::from_decoder_type(name).unwrap();
+    let codec = MediaCodec::from_decoder_type(name)?;
-    log::debug!("start init");
     let media_format = MediaFormat::new();
     media_format.set_str("mime", name);
     media_format.set_i32("width", 0);
     media_format.set_i32("height", 0);
     media_format.set_i32("color-format", 19); // COLOR_FormatYUV420Planar
     if let Err(e) = codec.configure(&media_format, None, direction) {
-        log::error!("failed to decoder.init:{:?}", e);
+        log::error!("Failed to init decoder:{:?}", e);
         return None;
     };
     log::error!("decoder init success");
     if let Err(e) = codec.start() {
-        log::error!("failed to decoder.start:{:?}", e);
+        log::error!("Failed to start decoder:{:?}", e);
         return None;
     };
-    log::debug!("init decoder successed!:{:?}", name);
+    log::debug!("Init decoder successed!: {:?}", name);
-    return Some(codec);
+    return Some(MediaCodecDecoder {
+        decoder: codec,
+        name: name.to_owned(),
+    });
+}
+
+pub fn check_mediacodec() {
+    std::thread::spawn(move || {
+        // check decoders
+        let decoders = MediaCodecDecoder::new_decoders();
+        H264_DECODER_SUPPORT.swap(decoders.h264.is_some(), Ordering::SeqCst);
+        H265_DECODER_SUPPORT.swap(decoders.h265.is_some(), Ordering::SeqCst);
+        decoders.h264.map(|d| d.stop());
+        decoders.h265.map(|d| d.stop());
+        // TODO encoders
+    });
 }

libs/scrap/src/common/mod.rs

@@ -32,6 +32,8 @@ pub mod codec;
 mod convert;
 #[cfg(feature = "hwcodec")]
 pub mod hwcodec;
+#[cfg(feature = "mediacodec")]
+pub mod mediacodec;
 pub mod vpxcodec;
 pub use self::convert::*;
 pub const STRIDE_ALIGN: usize = 64; // commonly used in libvpx vpx_img_alloc caller

src/flutter_ffi.rs

@@ -49,6 +49,8 @@ fn initialize(app_dir: &str) {
                 .with_min_level(log::Level::Debug) // limit log level
                 .with_tag("ffi"), // logs will show under mytag tag
         );
+        #[cfg(feature = "mediacodec")]
+        scrap::mediacodec::check_mediacodec();
     }
     #[cfg(target_os = "ios")]
     {