diff --git a/libs/hbb_common/protos/message.proto b/libs/hbb_common/protos/message.proto index 40da75a5d..1276d880e 100644 --- a/libs/hbb_common/protos/message.proto +++ b/libs/hbb_common/protos/message.proto @@ -1,33 +1,13 @@ syntax = "proto3"; package hbb; -message VP9 { +message EncodedVideoFrame { bytes data = 1; bool key = 2; int64 pts = 3; } -message VP9s { repeated VP9 frames = 1; } - -message H264 { - bytes data = 1; - bool key = 2; - int64 pts = 3; -} - -message H264s { - repeated H264 h264s = 1; -} - -message H265 { - bytes data = 1; - bool key = 2; - int64 pts = 3; -} - -message H265s { - repeated H265 h265s = 1; -} +message EncodedVideoFrames { repeated EncodedVideoFrame frames = 1; } message RGB { bool compress = 1; } @@ -39,11 +19,11 @@ message YUV { message VideoFrame { oneof union { - VP9s vp9s = 6; + EncodedVideoFrames vp9s = 6; RGB rgb = 7; YUV yuv = 8; - H264s h264s = 10; - H265s h265s = 11; + EncodedVideoFrames h264s = 10; + EncodedVideoFrames h265s = 11; } int64 timestamp = 9; } diff --git a/libs/scrap/src/common/codec.rs b/libs/scrap/src/common/codec.rs index 2a05c7326..fb5fc7b6f 100644 --- a/libs/scrap/src/common/codec.rs +++ b/libs/scrap/src/common/codec.rs @@ -12,15 +12,11 @@ use crate::vpxcodec::*; use hbb_common::{ anyhow::anyhow, log, - message_proto::{test_delay, video_frame, Message, VP9s, VideoCodecState}, + message_proto::{test_delay, video_frame, EncodedVideoFrames, Message, VideoCodecState}, ResultType, }; #[cfg(feature = "hwcodec")] -use hbb_common::{ - config::Config2, - lazy_static, - message_proto::{H264s, H265s}, -}; +use hbb_common::{config::Config2, lazy_static}; #[cfg(feature = "hwcodec")] lazy_static::lazy_static! { @@ -263,7 +259,7 @@ impl Decoder { #[cfg(feature = "hwcodec")] video_frame::Union::h264s(h264s) => { if let Some(decoder) = &mut self.hw.h264 { - Decoder::handle_h264s_video_frame(decoder, h264s, rgb, &mut self.i420) + Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420) } else { Err(anyhow!("don't support h264!")) } @@ -271,7 +267,7 @@ impl Decoder { #[cfg(feature = "hwcodec")] video_frame::Union::h265s(h265s) => { if let Some(decoder) = &mut self.hw.h265 { - Decoder::handle_h265s_video_frame(decoder, h265s, rgb, &mut self.i420) + Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420) } else { Err(anyhow!("don't support h265!")) } @@ -282,7 +278,7 @@ impl Decoder { fn handle_vp9s_video_frame( decoder: &mut VpxDecoder, - vp9s: &VP9s, + vp9s: &EncodedVideoFrames, rgb: &mut Vec, ) -> ResultType { let mut last_frame = Image::new(); @@ -305,14 +301,14 @@ impl Decoder { } #[cfg(feature = "hwcodec")] - fn handle_h264s_video_frame( + fn handle_hw_video_frame( decoder: &mut HwDecoder, - h264s: &H264s, + frames: &EncodedVideoFrames, rgb: &mut Vec, i420: &mut Vec, ) -> ResultType { let mut ret = false; - for h264 in h264s.h264s.iter() { + for h264 in frames.frames.iter() { for image in decoder.decode(&h264.data)? { // TODO: just process the last frame if image.bgra(rgb, i420).is_ok() { @@ -322,25 +318,6 @@ impl Decoder { } return Ok(ret); } - - #[cfg(feature = "hwcodec")] - fn handle_h265s_video_frame( - decoder: &mut HwDecoder, - h265s: &H265s, - rgb: &mut Vec, - i420: &mut Vec, - ) -> ResultType { - let mut ret = false; - for h265 in h265s.h265s.iter() { - for image in decoder.decode(&h265.data)? { - // TODO: just process the last frame - if image.bgra(rgb, i420).is_ok() { - ret = true; - } - } - } - return Ok(ret); - } } #[cfg(feature = "hwcodec")] diff --git a/libs/scrap/src/common/hwcodec.rs b/libs/scrap/src/common/hwcodec.rs index 70d944a33..8fd78e415 100644 --- a/libs/scrap/src/common/hwcodec.rs +++ b/libs/scrap/src/common/hwcodec.rs @@ -6,7 +6,7 @@ use hbb_common::{ anyhow::{anyhow, Context}, config::HwCodecConfig, lazy_static, log, - message_proto::{test_delay, H264s, H265s, Message, VideoFrame, H264, H265}, + message_proto::{test_delay, EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}, ResultType, }; use hwcodec::{ @@ -91,47 +91,27 @@ impl EncoderApi for HwEncoder { ) -> ResultType { let mut msg_out = Message::new(); let mut vf = VideoFrame::new(); - match self.format { - DataFormat::H264 => { - let mut h264s = Vec::new(); - for frame in self.encode(frame).with_context(|| "Failed to encode")? { - h264s.push(H264 { - data: frame.data, - pts: frame.pts as _, - ..Default::default() - }); - } - if h264s.len() > 0 { - vf.set_h264s(H264s { - h264s: h264s.into(), - ..Default::default() - }); - msg_out.set_video_frame(vf); - Ok(msg_out) - } else { - Err(anyhow!("no valid frame")) - } - } - DataFormat::H265 => { - let mut h265s = Vec::new(); - for frame in self.encode(frame).with_context(|| "Failed to encode")? { - h265s.push(H265 { - data: frame.data, - pts: frame.pts, - ..Default::default() - }); - } - if h265s.len() > 0 { - vf.set_h265s(H265s { - h265s, - ..Default::default() - }); - msg_out.set_video_frame(vf); - Ok(msg_out) - } else { - Err(anyhow!("no valid frame")) - } + let mut frames = Vec::new(); + for frame in self.encode(frame).with_context(|| "Failed to encode")? { + frames.push(EncodedVideoFrame { + data: frame.data, + pts: frame.pts as _, + ..Default::default() + }); + } + if frames.len() > 0 { + let frames = EncodedVideoFrames { + frames: frames.into(), + ..Default::default() + }; + match self.format { + DataFormat::H264 => vf.set_h264s(frames), + DataFormat::H265 => vf.set_h265s(frames), } + msg_out.set_video_frame(vf); + Ok(msg_out) + } else { + Err(anyhow!("no valid frame")) } } diff --git a/libs/scrap/src/common/vpxcodec.rs b/libs/scrap/src/common/vpxcodec.rs index b9b95b93e..c65bc43ab 100644 --- a/libs/scrap/src/common/vpxcodec.rs +++ b/libs/scrap/src/common/vpxcodec.rs @@ -3,7 +3,9 @@ // https://github.com/rust-av/vpx-rs/blob/master/src/decoder.rs use hbb_common::anyhow::{anyhow, Context}; -use hbb_common::message_proto::{test_delay, Message, VP9s, VideoFrame, VP9}; +use hbb_common::message_proto::{ + test_delay, EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame, +}; use hbb_common::ResultType; use crate::codec::EncoderApi; @@ -289,10 +291,10 @@ impl VpxEncoder { } #[inline] - fn create_msg(vp9s: Vec) -> Message { + fn create_msg(vp9s: Vec) -> Message { let mut msg_out = Message::new(); let mut vf = VideoFrame::new(); - vf.set_vp9s(VP9s { + vf.set_vp9s(EncodedVideoFrames { frames: vp9s.into(), ..Default::default() }); @@ -301,8 +303,8 @@ impl VpxEncoder { } #[inline] - fn create_frame(frame: &EncodeFrame) -> VP9 { - VP9 { + fn create_frame(frame: &EncodeFrame) -> EncodedVideoFrame { + EncodedVideoFrame { data: frame.data.to_vec(), key: frame.key, pts: frame.pts, diff --git a/src/server/video_service.rs b/src/server/video_service.rs index 355a43305..5815ab8b3 100644 --- a/src/server/video_service.rs +++ b/src/server/video_service.rs @@ -796,10 +796,10 @@ fn check_privacy_mode_changed(sp: &GenericService, privacy_mode_id: i32) -> Resu #[inline] #[cfg(any(target_os = "android", target_os = "ios"))] -fn create_msg(vp9s: Vec) -> Message { +fn create_msg(vp9s: Vec) -> Message { let mut msg_out = Message::new(); let mut vf = VideoFrame::new(); - vf.set_vp9s(VP9s { + vf.set_vp9s(EncodedVideoFrames { frames: vp9s.into(), ..Default::default() }); @@ -845,7 +845,7 @@ pub fn handle_one_frame_encoded( Ok(()) })?; let mut send_conn_ids: HashSet = Default::default(); - let vp9_frame = VP9 { + let vp9_frame = EncodedVideoFrame { data: frame.to_vec(), key: true, pts: ms,