hongshaorou/rustdesk
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

flutter texture render, mid commit

Browse Source 
Signed-off-by: fufesou <shuanglongchen@yeah.net>
This commit is contained in:
fufesou 2023-02-21 23:46:13 +08:00
parent 5acedecf0c
commit 77c4a14845
11 changed files with 322 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

61
flutter/lib/desktop/pages/remote_page.dart
View File

@ -65,6 +65,7 @@ class _RemotePageState extends State<RemotePage>
late RxBool _keyboardEnabled; late RxBool _keyboardEnabled;
late RxInt _textureId; late RxInt _textureId;
late int _textureKey; late int _textureKey;
final useTextureRender = bind.mainUseTextureRender();
final _blockableOverlayState = BlockableOverlayState(); final _blockableOverlayState = BlockableOverlayState();
@ -363,6 +364,7 @@ class _RemotePageState extends State<RemotePage>
keyboardEnabled: _keyboardEnabled, keyboardEnabled: _keyboardEnabled,
remoteCursorMoved: _remoteCursorMoved, remoteCursorMoved: _remoteCursorMoved,
textureId: _textureId, textureId: _textureId,
useTextureRender: useTextureRender,
listenerBuilder: (child) => listenerBuilder: (child) =>
_buildRawPointerMouseRegion(child, enterView, leaveView), _buildRawPointerMouseRegion(child, enterView, leaveView),
); );
@ -401,6 +403,7 @@ class ImagePaint extends StatefulWidget {
final RxBool keyboardEnabled; final RxBool keyboardEnabled;
final RxBool remoteCursorMoved; final RxBool remoteCursorMoved;
final RxInt textureId; final RxInt textureId;
final bool useTextureRender;
final Widget Function(Widget)? listenerBuilder; final Widget Function(Widget)? listenerBuilder;
ImagePaint( ImagePaint(
@ -411,6 +414,7 @@ class ImagePaint extends StatefulWidget {
required this.keyboardEnabled, required this.keyboardEnabled,
required this.remoteCursorMoved, required this.remoteCursorMoved,
required this.textureId, required this.textureId,
required this.useTextureRender,
this.listenerBuilder}) this.listenerBuilder})
: super(key: key); : super(key: key);
@ -485,15 +489,19 @@ class _ImagePaintState extends State<ImagePaint> {
final imageWidth = c.getDisplayWidth() * s; final imageWidth = c.getDisplayWidth() * s;
final imageHeight = c.getDisplayHeight() * s; final imageHeight = c.getDisplayHeight() * s;
final imageSize = Size(imageWidth, imageHeight); final imageSize = Size(imageWidth, imageHeight);
// final imageWidget = CustomPaint( late final Widget imageWidget;
// size: imageSize, if (widget.useTextureRender) {
// painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s), imageWidget = SizedBox(
// ); width: imageWidth,
final imageWidget = SizedBox( height: imageHeight,
width: imageWidth, child: Obx(() => Texture(textureId: widget.textureId.value)),
height: imageHeight, );
child: Obx(() => Texture(textureId: widget.textureId.value)), } else {
); imageWidget = CustomPaint(
size: imageSize,
painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s),
);
}
return NotificationListener<ScrollNotification>( return NotificationListener<ScrollNotification>(
onNotification: (notification) { onNotification: (notification) {
@ -517,22 +525,27 @@ class _ImagePaintState extends State<ImagePaint> {
context, _buildListener(imageWidget), c.size, imageSize)), context, _buildListener(imageWidget), c.size, imageSize)),
)); ));
} else { } else {
// final imageWidget = CustomPaint( late final Widget imageWidget;
// size: Size(c.size.width, c.size.height),
// painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s),
// );
if (c.size.width > 0 && c.size.height > 0) { if (c.size.width > 0 && c.size.height > 0) {
final imageWidget = Stack( if (widget.useTextureRender) {
children: [ imageWidget = Stack(
Positioned( children: [
left: c.x, Positioned(
top: c.y, left: c.x,
width: c.getDisplayWidth() * s, top: c.y,
height: c.getDisplayHeight() * s, width: c.getDisplayWidth() * s,
child: Texture(textureId: widget.textureId.value), height: c.getDisplayHeight() * s,
) child: Texture(textureId: widget.textureId.value),
], )
); ],
);
} else {
imageWidget = CustomPaint(
size: Size(c.size.width, c.size.height),
painter:
ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s),
);
}
return mouseRegion(child: _buildListener(imageWidget)); return mouseRegion(child: _buildListener(imageWidget));
} else { } else {
return Container(); return Container();

23
flutter/lib/models/model.dart
View File

@ -1438,6 +1438,7 @@ class FFI {
final stream = bind.sessionStart(id: id); final stream = bind.sessionStart(id: id);
final cb = ffiModel.startEventListener(id); final cb = ffiModel.startEventListener(id);
() async { () async {
final useTextureRender = bind.mainUseTextureRender();
// Preserved for the rgba data. // Preserved for the rgba data.
await for (final message in stream) { await for (final message in stream) {
if (message is EventToUI_Event) { if (message is EventToUI_Event) {
@ -1451,17 +1452,7 @@ class FFI {
debugPrint('json.decode fail1(): $e, ${message.field0}'); debugPrint('json.decode fail1(): $e, ${message.field0}');
} }
} else if (message is EventToUI_Rgba) { } else if (message is EventToUI_Rgba) {
if (Platform.isAndroid || Platform.isIOS) { if (useTextureRender) {
// Fetch the image buffer from rust codes.
final sz = platformFFI.getRgbaSize(id);
if (sz == null || sz == 0) {
return;
}
final rgba = platformFFI.getRgba(id, sz);
if (rgba != null) {
imageModel.onRgba(rgba);
}
} else {
if (_waitForImage[id]!) { if (_waitForImage[id]!) {
_waitForImage[id] = false; _waitForImage[id] = false;
dialogManager.dismissAll(); dialogManager.dismissAll();
@ -1471,6 +1462,16 @@ class FFI {
await canvasModel.updateViewStyle(); await canvasModel.updateViewStyle();
await canvasModel.updateScrollStyle(); await canvasModel.updateScrollStyle();
} }
} else {
// Fetch the image buffer from rust codes.
final sz = platformFFI.getRgbaSize(id);
if (sz == null || sz == 0) {
return;
}
final rgba = platformFFI.getRgba(id, sz);
if (rgba != null) {
imageModel.onRgba(rgba);
}
} }
} }
} }

26
libs/scrap/src/common/codec.rs
View File

@ -11,7 +11,7 @@ use crate::hwcodec::*;
use crate::mediacodec::{ use crate::mediacodec::{
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT, MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
}; };
use crate::vpxcodec::*; use crate::{vpxcodec::*, ImageFormat};
use hbb_common::{ use hbb_common::{
anyhow::anyhow, anyhow::anyhow,
@ -306,16 +306,17 @@ impl Decoder {
pub fn handle_video_frame( pub fn handle_video_frame(
&mut self, &mut self,
frame: &video_frame::Union, frame: &video_frame::Union,
fmt: ImageFormat,
rgb: &mut Vec<u8>, rgb: &mut Vec<u8>,
) -> ResultType<bool> { ) -> ResultType<bool> {
match frame { match frame {
video_frame::Union::Vp9s(vp9s) => { video_frame::Union::Vp9s(vp9s) => {
Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, rgb) Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, fmt, rgb)
} }
#[cfg(feature = "hwcodec")] #[cfg(feature = "hwcodec")]
video_frame::Union::H264s(h264s) => { video_frame::Union::H264s(h264s) => {
if let Some(decoder) = &mut self.hw.h264 { if let Some(decoder) = &mut self.hw.h264 {
Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420) Decoder::handle_hw_video_frame(decoder, h264s, fmt, rgb, &mut self.i420)
} else { } else {
Err(anyhow!("don't support h264!")) Err(anyhow!("don't support h264!"))
} }
@ -323,7 +324,7 @@ impl Decoder {
#[cfg(feature = "hwcodec")] #[cfg(feature = "hwcodec")]
video_frame::Union::H265s(h265s) => { video_frame::Union::H265s(h265s) => {
if let Some(decoder) = &mut self.hw.h265 { if let Some(decoder) = &mut self.hw.h265 {
Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420) Decoder::handle_hw_video_frame(decoder, h265s, fmt, rgb, &mut self.i420)
} else { } else {
Err(anyhow!("don't support h265!")) Err(anyhow!("don't support h265!"))
} }
@ -331,7 +332,7 @@ impl Decoder {
#[cfg(feature = "mediacodec")] #[cfg(feature = "mediacodec")]
video_frame::Union::H264s(h264s) => { video_frame::Union::H264s(h264s) => {
if let Some(decoder) = &mut self.media_codec.h264 { if let Some(decoder) = &mut self.media_codec.h264 {
Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb) Decoder::handle_mediacodec_video_frame(decoder, h264s, fmt, rgb)
} else { } else {
Err(anyhow!("don't support h264!")) Err(anyhow!("don't support h264!"))
} }
@ -339,7 +340,7 @@ impl Decoder {
#[cfg(feature = "mediacodec")] #[cfg(feature = "mediacodec")]
video_frame::Union::H265s(h265s) => { video_frame::Union::H265s(h265s) => {
if let Some(decoder) = &mut self.media_codec.h265 { if let Some(decoder) = &mut self.media_codec.h265 {
Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb) Decoder::handle_mediacodec_video_frame(decoder, h265s, fmt, rgb)
} else { } else {
Err(anyhow!("don't support h265!")) Err(anyhow!("don't support h265!"))
} }
@ -351,6 +352,7 @@ impl Decoder {
fn handle_vp9s_video_frame( fn handle_vp9s_video_frame(
decoder: &mut VpxDecoder, decoder: &mut VpxDecoder,
vp9s: &EncodedVideoFrames, vp9s: &EncodedVideoFrames,
fmt: ImageFormat,
rgb: &mut Vec<u8>, rgb: &mut Vec<u8>,
) -> ResultType<bool> { ) -> ResultType<bool> {
let mut last_frame = Image::new(); let mut last_frame = Image::new();
@ -367,7 +369,7 @@ impl Decoder {
if last_frame.is_null() { if last_frame.is_null() {
Ok(false) Ok(false)
} else { } else {
last_frame.rgb(1, true, rgb); last_frame.to(fmt, 1, rgb);
Ok(true) Ok(true)
} }
} }
@ -376,14 +378,15 @@ impl Decoder {
fn handle_hw_video_frame( fn handle_hw_video_frame(
decoder: &mut HwDecoder, decoder: &mut HwDecoder,
frames: &EncodedVideoFrames, frames: &EncodedVideoFrames,
rgb: &mut Vec<u8>, fmt: ImageFormat,
raw: &mut Vec<u8>,
i420: &mut Vec<u8>, i420: &mut Vec<u8>,
) -> ResultType<bool> { ) -> ResultType<bool> {
let mut ret = false; let mut ret = false;
for h264 in frames.frames.iter() { for h264 in frames.frames.iter() {
for image in decoder.decode(&h264.data)? { for image in decoder.decode(&h264.data)? {
// TODO: just process the last frame // TODO: just process the last frame
if image.bgra(rgb, i420).is_ok() { if image.to_fmt(fmt, raw, i420).is_ok() {
ret = true; ret = true;
} }
} }
@ -395,11 +398,12 @@ impl Decoder {
fn handle_mediacodec_video_frame( fn handle_mediacodec_video_frame(
decoder: &mut MediaCodecDecoder, decoder: &mut MediaCodecDecoder,
frames: &EncodedVideoFrames, frames: &EncodedVideoFrames,
rgb: &mut Vec<u8>, fmt: ImageFormat,
raw: &mut Vec<u8>,
) -> ResultType<bool> { ) -> ResultType<bool> {
let mut ret = false; let mut ret = false;
for h264 in frames.frames.iter() { for h264 in frames.frames.iter() {
return decoder.decode(&h264.data, rgb); return decoder.decode(&h264.data, fmt, raw);
} }
return Ok(false); return Ok(false);
} }

158
libs/scrap/src/common/convert.rs
View File

@ -103,6 +103,19 @@ extern "C" {
height: c_int, height: c_int,
) -> c_int; ) -> c_int;
pub fn I420ToABGR(
src_y: *const u8,
src_stride_y: c_int,
src_u: *const u8,
src_stride_u: c_int,
src_v: *const u8,
src_stride_v: c_int,
dst_rgba: *mut u8,
dst_stride_rgba: c_int,
width: c_int,
height: c_int,
) -> c_int;
pub fn NV12ToARGB( pub fn NV12ToARGB(
src_y: *const u8, src_y: *const u8,
src_stride_y: c_int, src_stride_y: c_int,
@ -246,6 +259,7 @@ pub unsafe fn nv12_to_i420(
#[cfg(feature = "hwcodec")] #[cfg(feature = "hwcodec")]
pub mod hw { pub mod hw {
use hbb_common::{anyhow::anyhow, ResultType}; use hbb_common::{anyhow::anyhow, ResultType};
use crate::ImageFormat;
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat}; use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat};
@ -315,7 +329,8 @@ pub mod hw {
} }
#[cfg(target_os = "windows")] #[cfg(target_os = "windows")]
pub fn hw_nv12_to_bgra( pub fn hw_nv12_to(
fmt: ImageFormat,
width: usize, width: usize,
height: usize, height: usize,
src_y: &[u8], src_y: &[u8],
@ -355,18 +370,39 @@ pub mod hw {
width as _, width as _,
height as _, height as _,
); );
super::I420ToARGB( match fmt {
i420_offset_y, ImageFormat::ARGB => {
i420_stride_y, super::I420ToARGB(
i420_offset_u, i420_offset_y,
i420_stride_u, i420_stride_y,
i420_offset_v, i420_offset_u,
i420_stride_v, i420_stride_u,
dst.as_mut_ptr(), i420_offset_v,
(width * 4) as _, i420_stride_v,
width as _, dst.as_mut_ptr(),
height as _, (width * 4) as _,
); width as _,
height as _,
);
}
ImageFormat::ABGR => {
super::I420ToABGR(
i420_offset_y,
i420_stride_y,
i420_offset_u,
i420_stride_u,
i420_offset_v,
i420_stride_v,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
}
_ => {
return Err(anyhow!("unsupported image format"));
}
}
return Ok(()); return Ok(());
}; };
} }
@ -374,7 +410,8 @@ pub mod hw {
} }
#[cfg(not(target_os = "windows"))] #[cfg(not(target_os = "windows"))]
pub fn hw_nv12_to_bgra( pub fn hw_nv12_to(
fmt: ImageFormat,
width: usize, width: usize,
height: usize, height: usize,
src_y: &[u8], src_y: &[u8],
@ -387,23 +424,46 @@ pub mod hw {
) -> ResultType<()> { ) -> ResultType<()> {
dst.resize(width * height * 4, 0); dst.resize(width * height * 4, 0);
unsafe { unsafe {
match super::NV12ToARGB( match fmt {
src_y.as_ptr(), ImageFormat::ARGB => {
src_stride_y as _, match super::NV12ToARGB(
src_uv.as_ptr(), src_y.as_ptr(),
src_stride_uv as _, src_stride_y as _,
dst.as_mut_ptr(), src_uv.as_ptr(),
(width * 4) as _, src_stride_uv as _,
width as _, dst.as_mut_ptr(),
height as _, (width * 4) as _,
) { width as _,
0 => Ok(()), height as _,
_ => Err(anyhow!("NV12ToARGB failed")), ) {
0 => Ok(()),
_ => Err(anyhow!("NV12ToARGB failed")),
}
}
ImageFormat::ABGR => {
match super::NV12ToABGR(
src_y.as_ptr(),
src_stride_y as _,
src_uv.as_ptr(),
src_stride_uv as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
) {
0 => Ok(()),
_ => Err(anyhow!("NV12ToABGR failed")),
}
}
_ => {
Err(anyhow!("unsupported image format"));
}
} }
} }
} }
pub fn hw_i420_to_bgra( pub fn hw_i420_to(
fmt: ImageFormat,
width: usize, width: usize,
height: usize, height: usize,
src_y: &[u8], src_y: &[u8],
@ -419,18 +479,38 @@ pub mod hw {
let src_v = src_v.as_ptr(); let src_v = src_v.as_ptr();
dst.resize(width * height * 4, 0); dst.resize(width * height * 4, 0);
unsafe { unsafe {
super::I420ToARGB( match fmt {
src_y, ImageFormat::ARGB => {
src_stride_y as _, super::I420ToARGB(
src_u, src_y,
src_stride_u as _, src_stride_y as _,
src_v, src_u,
src_stride_v as _, src_stride_u as _,
dst.as_mut_ptr(), src_v,
(width * 4) as _, src_stride_v as _,
width as _, dst.as_mut_ptr(),
height as _, (width * 4) as _,
); width as _,
height as _,
);
}
ImageFormat::ABGR => {
super::I420ToABGR(
src_y,
src_stride_y as _,
src_u,
src_stride_u as _,
src_v,
src_stride_v as _,
dst.as_mut_ptr(),
(width * 4) as _,
width as _,
height as _,
);
}
_ => {
}
}
}; };
} }
} }

22
libs/scrap/src/common/hwcodec.rs
View File

@ -1,6 +1,6 @@
use crate::{ use crate::{
codec::{EncoderApi, EncoderCfg}, codec::{EncoderApi, EncoderCfg},
hw, HW_STRIDE_ALIGN, hw, ImageFormat, HW_STRIDE_ALIGN,
}; };
use hbb_common::{ use hbb_common::{
anyhow::{anyhow, Context}, anyhow::{anyhow, Context},
@ -236,22 +236,24 @@ pub struct HwDecoderImage<'a> {
} }
impl HwDecoderImage<'_> { impl HwDecoderImage<'_> {
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> { pub fn to_fmt(&self, fmt: ImageFormat, fmt_data: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
let frame = self.frame; let frame = self.frame;
match frame.pixfmt { match frame.pixfmt {
AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to_bgra( AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to(
fmt,
frame.width as _, frame.width as _,
frame.height as _, frame.height as _,
&frame.data[0], &frame.data[0],
&frame.data[1], &frame.data[1],
frame.linesize[0] as _, frame.linesize[0] as _,
frame.linesize[1] as _, frame.linesize[1] as _,
bgra, fmt_data,
i420, i420,
HW_STRIDE_ALIGN, HW_STRIDE_ALIGN,
), ),
AVPixelFormat::AV_PIX_FMT_YUV420P => { AVPixelFormat::AV_PIX_FMT_YUV420P => {
hw::hw_i420_to_bgra( hw::hw_i420_to(
fmt,
frame.width as _, frame.width as _,
frame.height as _, frame.height as _,
&frame.data[0], &frame.data[0],
@ -260,12 +262,20 @@ impl HwDecoderImage<'_> {
frame.linesize[0] as _, frame.linesize[0] as _,
frame.linesize[1] as _, frame.linesize[1] as _,
frame.linesize[2] as _, frame.linesize[2] as _,
bgra, fmt_data,
); );
return Ok(()); return Ok(());
} }
} }
} }
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
self.to_fmt(ImageFormat::ARGB, bgra, i420)
}
pub fn rgba(&self, rgba: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
self.to_fmt(ImageFormat::ABGR, rgba, i420)
}
} }
fn get_config(k: &str) -> ResultType<CodecInfos> { fn get_config(k: &str) -> ResultType<CodecInfos> {

52
libs/scrap/src/common/mediacodec.rs
View File

@ -8,9 +8,10 @@ use std::{
time::Duration, time::Duration,
}; };
use crate::ImageFormat;
use crate::{ use crate::{
codec::{EncoderApi, EncoderCfg}, codec::{EncoderApi, EncoderCfg},
I420ToARGB, I420ToABGR, I420ToARGB,
}; };
/// MediaCodec mime type name /// MediaCodec mime type name
@ -50,7 +51,7 @@ impl MediaCodecDecoder {
MediaCodecDecoders { h264, h265 } MediaCodecDecoders { h264, h265 }
} }
pub fn decode(&mut self, data: &[u8], rgb: &mut Vec<u8>) -> ResultType<bool> { pub fn decode(&mut self, data: &[u8], fmt: ImageFormat, raw: &mut Vec<u8>) -> ResultType<bool> {
match self.dequeue_input_buffer(Duration::from_millis(10))? { match self.dequeue_input_buffer(Duration::from_millis(10))? {
Some(mut input_buffer) => { Some(mut input_buffer) => {
let mut buf = input_buffer.buffer_mut(); let mut buf = input_buffer.buffer_mut();
@ -83,23 +84,44 @@ impl MediaCodecDecoder {
let bps = 4; let bps = 4;
let u = buf.len() * 2 / 3; let u = buf.len() * 2 / 3;
let v = buf.len() * 5 / 6; let v = buf.len() * 5 / 6;
rgb.resize(h * w * bps, 0); raw.resize(h * w * bps, 0);
let y_ptr = buf.as_ptr(); let y_ptr = buf.as_ptr();
let u_ptr = buf[u..].as_ptr(); let u_ptr = buf[u..].as_ptr();
let v_ptr = buf[v..].as_ptr(); let v_ptr = buf[v..].as_ptr();
unsafe { unsafe {
I420ToARGB( match fmt {
y_ptr, ImageFormat::ARGB => {
stride, I420ToARGB(
u_ptr, y_ptr,
stride / 2, stride,
v_ptr, u_ptr,
stride / 2, stride / 2,
rgb.as_mut_ptr(), v_ptr,
(w * bps) as _, stride / 2,
w as _, raw.as_mut_ptr(),
h as _, (w * bps) as _,
); w as _,
h as _,
);
}
ImageFormat::ARGB => {
I420ToABGR(
y_ptr,
stride,
u_ptr,
stride / 2,
v_ptr,
stride / 2,
raw.as_mut_ptr(),
(w * bps) as _,
w as _,
h as _,
);
}
_ => {
bail!("Unsupported image format");
}
}
} }
self.release_output_buffer(output_buffer, false)?; self.release_output_buffer(output_buffer, false)?;
Ok(true) Ok(true)

7
libs/scrap/src/common/mod.rs
View File

@ -43,6 +43,13 @@ pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer
pub mod record; pub mod record;
mod vpx; mod vpx;
#[derive(Copy, Clone)]
pub enum ImageFormat {
Raw,
ABGR,
ARGB,
}
#[inline] #[inline]
pub fn would_block_if_equal(old: &mut Vec<u8>, b: &[u8]) -> std::io::Result<()> { pub fn would_block_if_equal(old: &mut Vec<u8>, b: &[u8]) -> std::io::Result<()> {
// does this really help? // does this really help?

82
libs/scrap/src/common/vpxcodec.rs
View File

@ -6,8 +6,8 @@ use hbb_common::anyhow::{anyhow, Context};
use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame}; use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
use hbb_common::{get_time, ResultType}; use hbb_common::{get_time, ResultType};
use crate::codec::EncoderApi;
use crate::STRIDE_ALIGN; use crate::STRIDE_ALIGN;
use crate::{codec::EncoderApi, ImageFormat};
use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *}; use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
use hbb_common::bytes::Bytes; use hbb_common::bytes::Bytes;
@ -417,7 +417,7 @@ impl VpxDecoder {
Ok(Self { ctx }) Ok(Self { ctx })
} }
pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result<Vec<u8>> { pub fn decode2rgb(&mut self, data: &[u8], fmt: ImageFormat) -> Result<Vec<u8>> {
let mut img = Image::new(); let mut img = Image::new();
for frame in self.decode(data)? { for frame in self.decode(data)? {
drop(img); drop(img);
@ -431,7 +431,7 @@ impl VpxDecoder {
Ok(Vec::new()) Ok(Vec::new())
} else { } else {
let mut out = Default::default(); let mut out = Default::default();
img.rgb(1, rgba, &mut out); img.to(fmt, 1, &mut out);
Ok(out) Ok(out)
} }
} }
@ -539,40 +539,60 @@ impl Image {
self.inner().stride[iplane] self.inner().stride[iplane]
} }
pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec<u8>) { pub fn to(&self, fmt: ImageFormat, stride_align: usize, dst: &mut Vec<u8>) {
let h = self.height(); let h = self.height();
let mut w = self.width(); let mut w = self.width();
let bps = if rgba { 4 } else { 3 }; let bps = match fmt {
ImageFormat::Raw => 3,
ImageFormat::ARGB | ImageFormat::ABGR => 4,
};
w = (w + stride_align - 1) & !(stride_align - 1); w = (w + stride_align - 1) & !(stride_align - 1);
dst.resize(h * w * bps, 0); dst.resize(h * w * bps, 0);
let img = self.inner(); let img = self.inner();
unsafe { unsafe {
if rgba { match fmt {
super::I420ToARGB( ImageFormat::Raw => {
img.planes[0], super::I420ToRAW(
img.stride[0], img.planes[0],
img.planes[1], img.stride[0],
img.stride[1], img.planes[1],
img.planes[2], img.stride[1],
img.stride[2], img.planes[2],
dst.as_mut_ptr(), img.stride[2],
(w * bps) as _, dst.as_mut_ptr(),
self.width() as _, (w * bps) as _,
self.height() as _, self.width() as _,
); self.height() as _,
} else { );
super::I420ToRAW( }
img.planes[0], ImageFormat::ARGB => {
img.stride[0], super::I420ToARGB(
img.planes[1], img.planes[0],
img.stride[1], img.stride[0],
img.planes[2], img.planes[1],
img.stride[2], img.stride[1],
dst.as_mut_ptr(), img.planes[2],
(w * bps) as _, img.stride[2],
self.width() as _, dst.as_mut_ptr(),
self.height() as _, (w * bps) as _,
); self.width() as _,
self.height() as _,
);
}
ImageFormat::ABGR => {
super::I420ToABGR(
img.planes[0],
img.stride[0],
img.planes[1],
img.stride[1],
img.planes[2],
img.stride[2],
dst.as_mut_ptr(),
(w * bps) as _,
self.width() as _,
self.height() as _,
);
}
} }
} }
} }

7
src/client.rs
View File

@ -45,6 +45,7 @@ use scrap::{
codec::{Decoder, DecoderCfg}, codec::{Decoder, DecoderCfg},
record::{Recorder, RecorderContext}, record::{Recorder, RecorderContext},
VpxDecoderConfig, VpxVideoCodecId, VpxDecoderConfig, VpxVideoCodecId,
ImageFormat,
}; };
use crate::{ use crate::{
@ -943,7 +944,11 @@ impl VideoHandler {
} }
match &vf.union { match &vf.union {
Some(frame) => { Some(frame) => {
let res = self.decoder.handle_video_frame(frame, &mut self.rgb); #[cfg(feature = "flutter_texture_render")]
let fmt = ImageFormat::ARGB;
#[cfg(not(feature = "flutter_texture_render"))]
let fmt = ImageFormat::ABGR;
let res = self.decoder.handle_video_frame(frame, fmt, &mut self.rgb);
if self.record { if self.record {
self.recorder self.recorder
.lock() .lock()

16
src/flutter.rs
View File

@ -128,19 +128,21 @@ pub struct FlutterHandler {
pub event_stream: Arc<RwLock<Option<StreamSink<EventToUI>>>>, pub event_stream: Arc<RwLock<Option<StreamSink<EventToUI>>>>,
// SAFETY: [rgba] is guarded by [rgba_valid], and it's safe to reach [rgba] with `rgba_valid == true`. // SAFETY: [rgba] is guarded by [rgba_valid], and it's safe to reach [rgba] with `rgba_valid == true`.
// We must check the `rgba_valid` before reading [rgba]. // We must check the `rgba_valid` before reading [rgba].
#[cfg(any(target_os = "android", target_os = "ios"))] #[cfg(not(feature = "flutter_texture_render"))]
pub rgba: Arc<RwLock<Vec<u8>>>, pub rgba: Arc<RwLock<Vec<u8>>>,
#[cfg(any(target_os = "android", target_os = "ios"))] #[cfg(not(feature = "flutter_texture_render"))]
pub rgba_valid: Arc<AtomicBool>, pub rgba_valid: Arc<AtomicBool>,
#[cfg(not(any(target_os = "android", target_os = "ios")))] #[cfg(feature = "flutter_texture_render")]
notify_rendered: Arc<RwLock<bool>>, notify_rendered: Arc<RwLock<bool>>,
renderer: Arc<RwLock<VideoRenderer>>, renderer: Arc<RwLock<VideoRenderer>>,
peer_info: Arc<RwLock<PeerInfo>>, peer_info: Arc<RwLock<PeerInfo>>,
} }
#[cfg(feature = "flutter_texture_render")]
pub type FlutterRgbaRendererPluginOnRgba = pub type FlutterRgbaRendererPluginOnRgba =
unsafe extern "C" fn(texture_rgba: *mut c_void, buffer: *const u8, width: c_int, height: c_int); unsafe extern "C" fn(texture_rgba: *mut c_void, buffer: *const u8, width: c_int, height: c_int);
// Video Texture Renderer in Flutter // Video Texture Renderer in Flutter
#[cfg(feature = "flutter_texture_render")]
#[derive(Clone)] #[derive(Clone)]
struct VideoRenderer { struct VideoRenderer {
// TextureRgba pointer in flutter native. // TextureRgba pointer in flutter native.
@ -151,6 +153,7 @@ struct VideoRenderer {
on_rgba_func: Symbol<'static, FlutterRgbaRendererPluginOnRgba>, on_rgba_func: Symbol<'static, FlutterRgbaRendererPluginOnRgba>,
} }
#[cfg(feature = "flutter_texture_render")]
impl Default for VideoRenderer { impl Default for VideoRenderer {
fn default() -> Self { fn default() -> Self {
unsafe { unsafe {
@ -167,6 +170,7 @@ impl Default for VideoRenderer {
} }
} }
#[cfg(feature = "flutter_texture_render")]
impl VideoRenderer { impl VideoRenderer {
#[inline] #[inline]
pub fn set_size(&mut self, width: i32, height: i32) { pub fn set_size(&mut self, width: i32, height: i32) {
@ -236,11 +240,13 @@ impl FlutterHandler {
} }
#[inline] #[inline]
#[cfg(feature = "flutter_texture_render")]
pub fn register_texture(&mut self, ptr: usize) { pub fn register_texture(&mut self, ptr: usize) {
self.renderer.write().unwrap().ptr = ptr; self.renderer.write().unwrap().ptr = ptr;
} }
#[inline] #[inline]
#[cfg(feature = "flutter_texture_render")]
pub fn set_size(&mut self, width: i32, height: i32) { pub fn set_size(&mut self, width: i32, height: i32) {
*self.notify_rendered.write().unwrap() = false; *self.notify_rendered.write().unwrap() = false;
self.renderer.write().unwrap().set_size(width, height); self.renderer.write().unwrap().set_size(width, height);
@ -405,7 +411,7 @@ impl InvokeUiSession for FlutterHandler {
fn adapt_size(&self) {} fn adapt_size(&self) {}
#[inline] #[inline]
#[cfg(any(target_os = "android", target_os = "ios"))] #[cfg(not(feature = "flutter_texture_render"))]
fn on_rgba(&self, data: &mut Vec<u8>) { fn on_rgba(&self, data: &mut Vec<u8>) {
// If the current rgba is not fetched by flutter, i.e., is valid. // If the current rgba is not fetched by flutter, i.e., is valid.
// We give up sending a new event to flutter. // We give up sending a new event to flutter.
@ -421,7 +427,7 @@ impl InvokeUiSession for FlutterHandler {
} }
#[inline] #[inline]
#[cfg(not(any(target_os = "android", target_os = "ios")))] #[cfg(feature = "flutter_texture_render")]
fn on_rgba(&self, data: &mut Vec<u8>) { fn on_rgba(&self, data: &mut Vec<u8>) {
self.renderer.read().unwrap().on_rgba(data); self.renderer.read().unwrap().on_rgba(data);
if *self.notify_rendered.read().unwrap() { if *self.notify_rendered.read().unwrap() {

11
src/flutter_ffi.rs
View File

@ -1306,6 +1306,17 @@ pub fn main_hide_docker() -> SyncReturn<bool> {
SyncReturn(true) SyncReturn(true)
} }
pub fn main_use_texture_render() -> SyncReturn<bool> {
#[cfg(not(feature = "flutter_texture_render"))]
{
SyncReturn(false)
}
#[cfg(feature = "flutter_texture_render")]
{
SyncReturn(true)
}
}
pub fn cm_start_listen_ipc_thread() { pub fn cm_start_listen_ipc_thread() {
#[cfg(not(any(target_os = "android", target_os = "ios")))] #[cfg(not(any(target_os = "android", target_os = "ios")))]
crate::flutter::connection_manager::start_listen_ipc_thread(); crate::flutter::connection_manager::start_listen_ipc_thread();