flutter texture render, mid commit
Signed-off-by: fufesou <shuanglongchen@yeah.net>
This commit is contained in:
parent
5acedecf0c
commit
77c4a14845
@ -65,6 +65,7 @@ class _RemotePageState extends State<RemotePage>
|
||||
late RxBool _keyboardEnabled;
|
||||
late RxInt _textureId;
|
||||
late int _textureKey;
|
||||
final useTextureRender = bind.mainUseTextureRender();
|
||||
|
||||
final _blockableOverlayState = BlockableOverlayState();
|
||||
|
||||
@ -363,6 +364,7 @@ class _RemotePageState extends State<RemotePage>
|
||||
keyboardEnabled: _keyboardEnabled,
|
||||
remoteCursorMoved: _remoteCursorMoved,
|
||||
textureId: _textureId,
|
||||
useTextureRender: useTextureRender,
|
||||
listenerBuilder: (child) =>
|
||||
_buildRawPointerMouseRegion(child, enterView, leaveView),
|
||||
);
|
||||
@ -401,6 +403,7 @@ class ImagePaint extends StatefulWidget {
|
||||
final RxBool keyboardEnabled;
|
||||
final RxBool remoteCursorMoved;
|
||||
final RxInt textureId;
|
||||
final bool useTextureRender;
|
||||
final Widget Function(Widget)? listenerBuilder;
|
||||
|
||||
ImagePaint(
|
||||
@ -411,6 +414,7 @@ class ImagePaint extends StatefulWidget {
|
||||
required this.keyboardEnabled,
|
||||
required this.remoteCursorMoved,
|
||||
required this.textureId,
|
||||
required this.useTextureRender,
|
||||
this.listenerBuilder})
|
||||
: super(key: key);
|
||||
|
||||
@ -485,15 +489,19 @@ class _ImagePaintState extends State<ImagePaint> {
|
||||
final imageWidth = c.getDisplayWidth() * s;
|
||||
final imageHeight = c.getDisplayHeight() * s;
|
||||
final imageSize = Size(imageWidth, imageHeight);
|
||||
// final imageWidget = CustomPaint(
|
||||
// size: imageSize,
|
||||
// painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s),
|
||||
// );
|
||||
final imageWidget = SizedBox(
|
||||
width: imageWidth,
|
||||
height: imageHeight,
|
||||
child: Obx(() => Texture(textureId: widget.textureId.value)),
|
||||
);
|
||||
late final Widget imageWidget;
|
||||
if (widget.useTextureRender) {
|
||||
imageWidget = SizedBox(
|
||||
width: imageWidth,
|
||||
height: imageHeight,
|
||||
child: Obx(() => Texture(textureId: widget.textureId.value)),
|
||||
);
|
||||
} else {
|
||||
imageWidget = CustomPaint(
|
||||
size: imageSize,
|
||||
painter: ImagePainter(image: m.image, x: 0, y: 0, scale: s),
|
||||
);
|
||||
}
|
||||
|
||||
return NotificationListener<ScrollNotification>(
|
||||
onNotification: (notification) {
|
||||
@ -517,22 +525,27 @@ class _ImagePaintState extends State<ImagePaint> {
|
||||
context, _buildListener(imageWidget), c.size, imageSize)),
|
||||
));
|
||||
} else {
|
||||
// final imageWidget = CustomPaint(
|
||||
// size: Size(c.size.width, c.size.height),
|
||||
// painter: ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s),
|
||||
// );
|
||||
late final Widget imageWidget;
|
||||
if (c.size.width > 0 && c.size.height > 0) {
|
||||
final imageWidget = Stack(
|
||||
children: [
|
||||
Positioned(
|
||||
left: c.x,
|
||||
top: c.y,
|
||||
width: c.getDisplayWidth() * s,
|
||||
height: c.getDisplayHeight() * s,
|
||||
child: Texture(textureId: widget.textureId.value),
|
||||
)
|
||||
],
|
||||
);
|
||||
if (widget.useTextureRender) {
|
||||
imageWidget = Stack(
|
||||
children: [
|
||||
Positioned(
|
||||
left: c.x,
|
||||
top: c.y,
|
||||
width: c.getDisplayWidth() * s,
|
||||
height: c.getDisplayHeight() * s,
|
||||
child: Texture(textureId: widget.textureId.value),
|
||||
)
|
||||
],
|
||||
);
|
||||
} else {
|
||||
imageWidget = CustomPaint(
|
||||
size: Size(c.size.width, c.size.height),
|
||||
painter:
|
||||
ImagePainter(image: m.image, x: c.x / s, y: c.y / s, scale: s),
|
||||
);
|
||||
}
|
||||
return mouseRegion(child: _buildListener(imageWidget));
|
||||
} else {
|
||||
return Container();
|
||||
|
@ -1438,6 +1438,7 @@ class FFI {
|
||||
final stream = bind.sessionStart(id: id);
|
||||
final cb = ffiModel.startEventListener(id);
|
||||
() async {
|
||||
final useTextureRender = bind.mainUseTextureRender();
|
||||
// Preserved for the rgba data.
|
||||
await for (final message in stream) {
|
||||
if (message is EventToUI_Event) {
|
||||
@ -1451,17 +1452,7 @@ class FFI {
|
||||
debugPrint('json.decode fail1(): $e, ${message.field0}');
|
||||
}
|
||||
} else if (message is EventToUI_Rgba) {
|
||||
if (Platform.isAndroid || Platform.isIOS) {
|
||||
// Fetch the image buffer from rust codes.
|
||||
final sz = platformFFI.getRgbaSize(id);
|
||||
if (sz == null || sz == 0) {
|
||||
return;
|
||||
}
|
||||
final rgba = platformFFI.getRgba(id, sz);
|
||||
if (rgba != null) {
|
||||
imageModel.onRgba(rgba);
|
||||
}
|
||||
} else {
|
||||
if (useTextureRender) {
|
||||
if (_waitForImage[id]!) {
|
||||
_waitForImage[id] = false;
|
||||
dialogManager.dismissAll();
|
||||
@ -1471,6 +1462,16 @@ class FFI {
|
||||
await canvasModel.updateViewStyle();
|
||||
await canvasModel.updateScrollStyle();
|
||||
}
|
||||
} else {
|
||||
// Fetch the image buffer from rust codes.
|
||||
final sz = platformFFI.getRgbaSize(id);
|
||||
if (sz == null || sz == 0) {
|
||||
return;
|
||||
}
|
||||
final rgba = platformFFI.getRgba(id, sz);
|
||||
if (rgba != null) {
|
||||
imageModel.onRgba(rgba);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -11,7 +11,7 @@ use crate::hwcodec::*;
|
||||
use crate::mediacodec::{
|
||||
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
|
||||
};
|
||||
use crate::vpxcodec::*;
|
||||
use crate::{vpxcodec::*, ImageFormat};
|
||||
|
||||
use hbb_common::{
|
||||
anyhow::anyhow,
|
||||
@ -306,16 +306,17 @@ impl Decoder {
|
||||
pub fn handle_video_frame(
|
||||
&mut self,
|
||||
frame: &video_frame::Union,
|
||||
fmt: ImageFormat,
|
||||
rgb: &mut Vec<u8>,
|
||||
) -> ResultType<bool> {
|
||||
match frame {
|
||||
video_frame::Union::Vp9s(vp9s) => {
|
||||
Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, rgb)
|
||||
Decoder::handle_vp9s_video_frame(&mut self.vpx, vp9s, fmt, rgb)
|
||||
}
|
||||
#[cfg(feature = "hwcodec")]
|
||||
video_frame::Union::H264s(h264s) => {
|
||||
if let Some(decoder) = &mut self.hw.h264 {
|
||||
Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420)
|
||||
Decoder::handle_hw_video_frame(decoder, h264s, fmt, rgb, &mut self.i420)
|
||||
} else {
|
||||
Err(anyhow!("don't support h264!"))
|
||||
}
|
||||
@ -323,7 +324,7 @@ impl Decoder {
|
||||
#[cfg(feature = "hwcodec")]
|
||||
video_frame::Union::H265s(h265s) => {
|
||||
if let Some(decoder) = &mut self.hw.h265 {
|
||||
Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420)
|
||||
Decoder::handle_hw_video_frame(decoder, h265s, fmt, rgb, &mut self.i420)
|
||||
} else {
|
||||
Err(anyhow!("don't support h265!"))
|
||||
}
|
||||
@ -331,7 +332,7 @@ impl Decoder {
|
||||
#[cfg(feature = "mediacodec")]
|
||||
video_frame::Union::H264s(h264s) => {
|
||||
if let Some(decoder) = &mut self.media_codec.h264 {
|
||||
Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
|
||||
Decoder::handle_mediacodec_video_frame(decoder, h264s, fmt, rgb)
|
||||
} else {
|
||||
Err(anyhow!("don't support h264!"))
|
||||
}
|
||||
@ -339,7 +340,7 @@ impl Decoder {
|
||||
#[cfg(feature = "mediacodec")]
|
||||
video_frame::Union::H265s(h265s) => {
|
||||
if let Some(decoder) = &mut self.media_codec.h265 {
|
||||
Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
|
||||
Decoder::handle_mediacodec_video_frame(decoder, h265s, fmt, rgb)
|
||||
} else {
|
||||
Err(anyhow!("don't support h265!"))
|
||||
}
|
||||
@ -351,6 +352,7 @@ impl Decoder {
|
||||
fn handle_vp9s_video_frame(
|
||||
decoder: &mut VpxDecoder,
|
||||
vp9s: &EncodedVideoFrames,
|
||||
fmt: ImageFormat,
|
||||
rgb: &mut Vec<u8>,
|
||||
) -> ResultType<bool> {
|
||||
let mut last_frame = Image::new();
|
||||
@ -367,7 +369,7 @@ impl Decoder {
|
||||
if last_frame.is_null() {
|
||||
Ok(false)
|
||||
} else {
|
||||
last_frame.rgb(1, true, rgb);
|
||||
last_frame.to(fmt, 1, rgb);
|
||||
Ok(true)
|
||||
}
|
||||
}
|
||||
@ -376,14 +378,15 @@ impl Decoder {
|
||||
fn handle_hw_video_frame(
|
||||
decoder: &mut HwDecoder,
|
||||
frames: &EncodedVideoFrames,
|
||||
rgb: &mut Vec<u8>,
|
||||
fmt: ImageFormat,
|
||||
raw: &mut Vec<u8>,
|
||||
i420: &mut Vec<u8>,
|
||||
) -> ResultType<bool> {
|
||||
let mut ret = false;
|
||||
for h264 in frames.frames.iter() {
|
||||
for image in decoder.decode(&h264.data)? {
|
||||
// TODO: just process the last frame
|
||||
if image.bgra(rgb, i420).is_ok() {
|
||||
if image.to_fmt(fmt, raw, i420).is_ok() {
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
@ -395,11 +398,12 @@ impl Decoder {
|
||||
fn handle_mediacodec_video_frame(
|
||||
decoder: &mut MediaCodecDecoder,
|
||||
frames: &EncodedVideoFrames,
|
||||
rgb: &mut Vec<u8>,
|
||||
fmt: ImageFormat,
|
||||
raw: &mut Vec<u8>,
|
||||
) -> ResultType<bool> {
|
||||
let mut ret = false;
|
||||
for h264 in frames.frames.iter() {
|
||||
return decoder.decode(&h264.data, rgb);
|
||||
return decoder.decode(&h264.data, fmt, raw);
|
||||
}
|
||||
return Ok(false);
|
||||
}
|
||||
|
@ -103,6 +103,19 @@ extern "C" {
|
||||
height: c_int,
|
||||
) -> c_int;
|
||||
|
||||
pub fn I420ToABGR(
|
||||
src_y: *const u8,
|
||||
src_stride_y: c_int,
|
||||
src_u: *const u8,
|
||||
src_stride_u: c_int,
|
||||
src_v: *const u8,
|
||||
src_stride_v: c_int,
|
||||
dst_rgba: *mut u8,
|
||||
dst_stride_rgba: c_int,
|
||||
width: c_int,
|
||||
height: c_int,
|
||||
) -> c_int;
|
||||
|
||||
pub fn NV12ToARGB(
|
||||
src_y: *const u8,
|
||||
src_stride_y: c_int,
|
||||
@ -246,6 +259,7 @@ pub unsafe fn nv12_to_i420(
|
||||
#[cfg(feature = "hwcodec")]
|
||||
pub mod hw {
|
||||
use hbb_common::{anyhow::anyhow, ResultType};
|
||||
use crate::ImageFormat;
|
||||
#[cfg(target_os = "windows")]
|
||||
use hwcodec::{ffmpeg::ffmpeg_linesize_offset_length, AVPixelFormat};
|
||||
|
||||
@ -315,7 +329,8 @@ pub mod hw {
|
||||
}
|
||||
|
||||
#[cfg(target_os = "windows")]
|
||||
pub fn hw_nv12_to_bgra(
|
||||
pub fn hw_nv12_to(
|
||||
fmt: ImageFormat,
|
||||
width: usize,
|
||||
height: usize,
|
||||
src_y: &[u8],
|
||||
@ -355,18 +370,39 @@ pub mod hw {
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
super::I420ToARGB(
|
||||
i420_offset_y,
|
||||
i420_stride_y,
|
||||
i420_offset_u,
|
||||
i420_stride_u,
|
||||
i420_offset_v,
|
||||
i420_stride_v,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
match fmt {
|
||||
ImageFormat::ARGB => {
|
||||
super::I420ToARGB(
|
||||
i420_offset_y,
|
||||
i420_stride_y,
|
||||
i420_offset_u,
|
||||
i420_stride_u,
|
||||
i420_offset_v,
|
||||
i420_stride_v,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
}
|
||||
ImageFormat::ABGR => {
|
||||
super::I420ToABGR(
|
||||
i420_offset_y,
|
||||
i420_stride_y,
|
||||
i420_offset_u,
|
||||
i420_stride_u,
|
||||
i420_offset_v,
|
||||
i420_stride_v,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
return Err(anyhow!("unsupported image format"));
|
||||
}
|
||||
}
|
||||
return Ok(());
|
||||
};
|
||||
}
|
||||
@ -374,7 +410,8 @@ pub mod hw {
|
||||
}
|
||||
|
||||
#[cfg(not(target_os = "windows"))]
|
||||
pub fn hw_nv12_to_bgra(
|
||||
pub fn hw_nv12_to(
|
||||
fmt: ImageFormat,
|
||||
width: usize,
|
||||
height: usize,
|
||||
src_y: &[u8],
|
||||
@ -387,23 +424,46 @@ pub mod hw {
|
||||
) -> ResultType<()> {
|
||||
dst.resize(width * height * 4, 0);
|
||||
unsafe {
|
||||
match super::NV12ToARGB(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as _,
|
||||
src_uv.as_ptr(),
|
||||
src_stride_uv as _,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
) {
|
||||
0 => Ok(()),
|
||||
_ => Err(anyhow!("NV12ToARGB failed")),
|
||||
match fmt {
|
||||
ImageFormat::ARGB => {
|
||||
match super::NV12ToARGB(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as _,
|
||||
src_uv.as_ptr(),
|
||||
src_stride_uv as _,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
) {
|
||||
0 => Ok(()),
|
||||
_ => Err(anyhow!("NV12ToARGB failed")),
|
||||
}
|
||||
}
|
||||
ImageFormat::ABGR => {
|
||||
match super::NV12ToABGR(
|
||||
src_y.as_ptr(),
|
||||
src_stride_y as _,
|
||||
src_uv.as_ptr(),
|
||||
src_stride_uv as _,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
) {
|
||||
0 => Ok(()),
|
||||
_ => Err(anyhow!("NV12ToABGR failed")),
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
Err(anyhow!("unsupported image format"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hw_i420_to_bgra(
|
||||
pub fn hw_i420_to(
|
||||
fmt: ImageFormat,
|
||||
width: usize,
|
||||
height: usize,
|
||||
src_y: &[u8],
|
||||
@ -419,18 +479,38 @@ pub mod hw {
|
||||
let src_v = src_v.as_ptr();
|
||||
dst.resize(width * height * 4, 0);
|
||||
unsafe {
|
||||
super::I420ToARGB(
|
||||
src_y,
|
||||
src_stride_y as _,
|
||||
src_u,
|
||||
src_stride_u as _,
|
||||
src_v,
|
||||
src_stride_v as _,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
match fmt {
|
||||
ImageFormat::ARGB => {
|
||||
super::I420ToARGB(
|
||||
src_y,
|
||||
src_stride_y as _,
|
||||
src_u,
|
||||
src_stride_u as _,
|
||||
src_v,
|
||||
src_stride_v as _,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
}
|
||||
ImageFormat::ABGR => {
|
||||
super::I420ToABGR(
|
||||
src_y,
|
||||
src_stride_y as _,
|
||||
src_u,
|
||||
src_stride_u as _,
|
||||
src_v,
|
||||
src_stride_v as _,
|
||||
dst.as_mut_ptr(),
|
||||
(width * 4) as _,
|
||||
width as _,
|
||||
height as _,
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
|
@ -1,6 +1,6 @@
|
||||
use crate::{
|
||||
codec::{EncoderApi, EncoderCfg},
|
||||
hw, HW_STRIDE_ALIGN,
|
||||
hw, ImageFormat, HW_STRIDE_ALIGN,
|
||||
};
|
||||
use hbb_common::{
|
||||
anyhow::{anyhow, Context},
|
||||
@ -236,22 +236,24 @@ pub struct HwDecoderImage<'a> {
|
||||
}
|
||||
|
||||
impl HwDecoderImage<'_> {
|
||||
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
|
||||
pub fn to_fmt(&self, fmt: ImageFormat, fmt_data: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
|
||||
let frame = self.frame;
|
||||
match frame.pixfmt {
|
||||
AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to_bgra(
|
||||
AVPixelFormat::AV_PIX_FMT_NV12 => hw::hw_nv12_to(
|
||||
fmt,
|
||||
frame.width as _,
|
||||
frame.height as _,
|
||||
&frame.data[0],
|
||||
&frame.data[1],
|
||||
frame.linesize[0] as _,
|
||||
frame.linesize[1] as _,
|
||||
bgra,
|
||||
fmt_data,
|
||||
i420,
|
||||
HW_STRIDE_ALIGN,
|
||||
),
|
||||
AVPixelFormat::AV_PIX_FMT_YUV420P => {
|
||||
hw::hw_i420_to_bgra(
|
||||
hw::hw_i420_to(
|
||||
fmt,
|
||||
frame.width as _,
|
||||
frame.height as _,
|
||||
&frame.data[0],
|
||||
@ -260,12 +262,20 @@ impl HwDecoderImage<'_> {
|
||||
frame.linesize[0] as _,
|
||||
frame.linesize[1] as _,
|
||||
frame.linesize[2] as _,
|
||||
bgra,
|
||||
fmt_data,
|
||||
);
|
||||
return Ok(());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn bgra(&self, bgra: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
|
||||
self.to_fmt(ImageFormat::ARGB, bgra, i420)
|
||||
}
|
||||
|
||||
pub fn rgba(&self, rgba: &mut Vec<u8>, i420: &mut Vec<u8>) -> ResultType<()> {
|
||||
self.to_fmt(ImageFormat::ABGR, rgba, i420)
|
||||
}
|
||||
}
|
||||
|
||||
fn get_config(k: &str) -> ResultType<CodecInfos> {
|
||||
|
@ -8,9 +8,10 @@ use std::{
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
use crate::ImageFormat;
|
||||
use crate::{
|
||||
codec::{EncoderApi, EncoderCfg},
|
||||
I420ToARGB,
|
||||
I420ToABGR, I420ToARGB,
|
||||
};
|
||||
|
||||
/// MediaCodec mime type name
|
||||
@ -50,7 +51,7 @@ impl MediaCodecDecoder {
|
||||
MediaCodecDecoders { h264, h265 }
|
||||
}
|
||||
|
||||
pub fn decode(&mut self, data: &[u8], rgb: &mut Vec<u8>) -> ResultType<bool> {
|
||||
pub fn decode(&mut self, data: &[u8], fmt: ImageFormat, raw: &mut Vec<u8>) -> ResultType<bool> {
|
||||
match self.dequeue_input_buffer(Duration::from_millis(10))? {
|
||||
Some(mut input_buffer) => {
|
||||
let mut buf = input_buffer.buffer_mut();
|
||||
@ -83,23 +84,44 @@ impl MediaCodecDecoder {
|
||||
let bps = 4;
|
||||
let u = buf.len() * 2 / 3;
|
||||
let v = buf.len() * 5 / 6;
|
||||
rgb.resize(h * w * bps, 0);
|
||||
raw.resize(h * w * bps, 0);
|
||||
let y_ptr = buf.as_ptr();
|
||||
let u_ptr = buf[u..].as_ptr();
|
||||
let v_ptr = buf[v..].as_ptr();
|
||||
unsafe {
|
||||
I420ToARGB(
|
||||
y_ptr,
|
||||
stride,
|
||||
u_ptr,
|
||||
stride / 2,
|
||||
v_ptr,
|
||||
stride / 2,
|
||||
rgb.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
w as _,
|
||||
h as _,
|
||||
);
|
||||
match fmt {
|
||||
ImageFormat::ARGB => {
|
||||
I420ToARGB(
|
||||
y_ptr,
|
||||
stride,
|
||||
u_ptr,
|
||||
stride / 2,
|
||||
v_ptr,
|
||||
stride / 2,
|
||||
raw.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
w as _,
|
||||
h as _,
|
||||
);
|
||||
}
|
||||
ImageFormat::ARGB => {
|
||||
I420ToABGR(
|
||||
y_ptr,
|
||||
stride,
|
||||
u_ptr,
|
||||
stride / 2,
|
||||
v_ptr,
|
||||
stride / 2,
|
||||
raw.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
w as _,
|
||||
h as _,
|
||||
);
|
||||
}
|
||||
_ => {
|
||||
bail!("Unsupported image format");
|
||||
}
|
||||
}
|
||||
}
|
||||
self.release_output_buffer(output_buffer, false)?;
|
||||
Ok(true)
|
||||
|
@ -43,6 +43,13 @@ pub const HW_STRIDE_ALIGN: usize = 0; // recommended by av_frame_get_buffer
|
||||
pub mod record;
|
||||
mod vpx;
|
||||
|
||||
#[derive(Copy, Clone)]
|
||||
pub enum ImageFormat {
|
||||
Raw,
|
||||
ABGR,
|
||||
ARGB,
|
||||
}
|
||||
|
||||
#[inline]
|
||||
pub fn would_block_if_equal(old: &mut Vec<u8>, b: &[u8]) -> std::io::Result<()> {
|
||||
// does this really help?
|
||||
|
@ -6,8 +6,8 @@ use hbb_common::anyhow::{anyhow, Context};
|
||||
use hbb_common::message_proto::{EncodedVideoFrame, EncodedVideoFrames, Message, VideoFrame};
|
||||
use hbb_common::{get_time, ResultType};
|
||||
|
||||
use crate::codec::EncoderApi;
|
||||
use crate::STRIDE_ALIGN;
|
||||
use crate::{codec::EncoderApi, ImageFormat};
|
||||
|
||||
use super::vpx::{vp8e_enc_control_id::*, vpx_codec_err_t::*, *};
|
||||
use hbb_common::bytes::Bytes;
|
||||
@ -417,7 +417,7 @@ impl VpxDecoder {
|
||||
Ok(Self { ctx })
|
||||
}
|
||||
|
||||
pub fn decode2rgb(&mut self, data: &[u8], rgba: bool) -> Result<Vec<u8>> {
|
||||
pub fn decode2rgb(&mut self, data: &[u8], fmt: ImageFormat) -> Result<Vec<u8>> {
|
||||
let mut img = Image::new();
|
||||
for frame in self.decode(data)? {
|
||||
drop(img);
|
||||
@ -431,7 +431,7 @@ impl VpxDecoder {
|
||||
Ok(Vec::new())
|
||||
} else {
|
||||
let mut out = Default::default();
|
||||
img.rgb(1, rgba, &mut out);
|
||||
img.to(fmt, 1, &mut out);
|
||||
Ok(out)
|
||||
}
|
||||
}
|
||||
@ -539,40 +539,60 @@ impl Image {
|
||||
self.inner().stride[iplane]
|
||||
}
|
||||
|
||||
pub fn rgb(&self, stride_align: usize, rgba: bool, dst: &mut Vec<u8>) {
|
||||
pub fn to(&self, fmt: ImageFormat, stride_align: usize, dst: &mut Vec<u8>) {
|
||||
let h = self.height();
|
||||
let mut w = self.width();
|
||||
let bps = if rgba { 4 } else { 3 };
|
||||
let bps = match fmt {
|
||||
ImageFormat::Raw => 3,
|
||||
ImageFormat::ARGB | ImageFormat::ABGR => 4,
|
||||
};
|
||||
w = (w + stride_align - 1) & !(stride_align - 1);
|
||||
dst.resize(h * w * bps, 0);
|
||||
let img = self.inner();
|
||||
unsafe {
|
||||
if rgba {
|
||||
super::I420ToARGB(
|
||||
img.planes[0],
|
||||
img.stride[0],
|
||||
img.planes[1],
|
||||
img.stride[1],
|
||||
img.planes[2],
|
||||
img.stride[2],
|
||||
dst.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
self.width() as _,
|
||||
self.height() as _,
|
||||
);
|
||||
} else {
|
||||
super::I420ToRAW(
|
||||
img.planes[0],
|
||||
img.stride[0],
|
||||
img.planes[1],
|
||||
img.stride[1],
|
||||
img.planes[2],
|
||||
img.stride[2],
|
||||
dst.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
self.width() as _,
|
||||
self.height() as _,
|
||||
);
|
||||
match fmt {
|
||||
ImageFormat::Raw => {
|
||||
super::I420ToRAW(
|
||||
img.planes[0],
|
||||
img.stride[0],
|
||||
img.planes[1],
|
||||
img.stride[1],
|
||||
img.planes[2],
|
||||
img.stride[2],
|
||||
dst.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
self.width() as _,
|
||||
self.height() as _,
|
||||
);
|
||||
}
|
||||
ImageFormat::ARGB => {
|
||||
super::I420ToARGB(
|
||||
img.planes[0],
|
||||
img.stride[0],
|
||||
img.planes[1],
|
||||
img.stride[1],
|
||||
img.planes[2],
|
||||
img.stride[2],
|
||||
dst.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
self.width() as _,
|
||||
self.height() as _,
|
||||
);
|
||||
}
|
||||
ImageFormat::ABGR => {
|
||||
super::I420ToABGR(
|
||||
img.planes[0],
|
||||
img.stride[0],
|
||||
img.planes[1],
|
||||
img.stride[1],
|
||||
img.planes[2],
|
||||
img.stride[2],
|
||||
dst.as_mut_ptr(),
|
||||
(w * bps) as _,
|
||||
self.width() as _,
|
||||
self.height() as _,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -45,6 +45,7 @@ use scrap::{
|
||||
codec::{Decoder, DecoderCfg},
|
||||
record::{Recorder, RecorderContext},
|
||||
VpxDecoderConfig, VpxVideoCodecId,
|
||||
ImageFormat,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@ -943,7 +944,11 @@ impl VideoHandler {
|
||||
}
|
||||
match &vf.union {
|
||||
Some(frame) => {
|
||||
let res = self.decoder.handle_video_frame(frame, &mut self.rgb);
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
let fmt = ImageFormat::ARGB;
|
||||
#[cfg(not(feature = "flutter_texture_render"))]
|
||||
let fmt = ImageFormat::ABGR;
|
||||
let res = self.decoder.handle_video_frame(frame, fmt, &mut self.rgb);
|
||||
if self.record {
|
||||
self.recorder
|
||||
.lock()
|
||||
|
@ -128,19 +128,21 @@ pub struct FlutterHandler {
|
||||
pub event_stream: Arc<RwLock<Option<StreamSink<EventToUI>>>>,
|
||||
// SAFETY: [rgba] is guarded by [rgba_valid], and it's safe to reach [rgba] with `rgba_valid == true`.
|
||||
// We must check the `rgba_valid` before reading [rgba].
|
||||
#[cfg(any(target_os = "android", target_os = "ios"))]
|
||||
#[cfg(not(feature = "flutter_texture_render"))]
|
||||
pub rgba: Arc<RwLock<Vec<u8>>>,
|
||||
#[cfg(any(target_os = "android", target_os = "ios"))]
|
||||
#[cfg(not(feature = "flutter_texture_render"))]
|
||||
pub rgba_valid: Arc<AtomicBool>,
|
||||
#[cfg(not(any(target_os = "android", target_os = "ios")))]
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
notify_rendered: Arc<RwLock<bool>>,
|
||||
renderer: Arc<RwLock<VideoRenderer>>,
|
||||
peer_info: Arc<RwLock<PeerInfo>>,
|
||||
}
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
pub type FlutterRgbaRendererPluginOnRgba =
|
||||
unsafe extern "C" fn(texture_rgba: *mut c_void, buffer: *const u8, width: c_int, height: c_int);
|
||||
|
||||
// Video Texture Renderer in Flutter
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
#[derive(Clone)]
|
||||
struct VideoRenderer {
|
||||
// TextureRgba pointer in flutter native.
|
||||
@ -151,6 +153,7 @@ struct VideoRenderer {
|
||||
on_rgba_func: Symbol<'static, FlutterRgbaRendererPluginOnRgba>,
|
||||
}
|
||||
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
impl Default for VideoRenderer {
|
||||
fn default() -> Self {
|
||||
unsafe {
|
||||
@ -167,6 +170,7 @@ impl Default for VideoRenderer {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
impl VideoRenderer {
|
||||
#[inline]
|
||||
pub fn set_size(&mut self, width: i32, height: i32) {
|
||||
@ -236,11 +240,13 @@ impl FlutterHandler {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
pub fn register_texture(&mut self, ptr: usize) {
|
||||
self.renderer.write().unwrap().ptr = ptr;
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
pub fn set_size(&mut self, width: i32, height: i32) {
|
||||
*self.notify_rendered.write().unwrap() = false;
|
||||
self.renderer.write().unwrap().set_size(width, height);
|
||||
@ -405,7 +411,7 @@ impl InvokeUiSession for FlutterHandler {
|
||||
fn adapt_size(&self) {}
|
||||
|
||||
#[inline]
|
||||
#[cfg(any(target_os = "android", target_os = "ios"))]
|
||||
#[cfg(not(feature = "flutter_texture_render"))]
|
||||
fn on_rgba(&self, data: &mut Vec<u8>) {
|
||||
// If the current rgba is not fetched by flutter, i.e., is valid.
|
||||
// We give up sending a new event to flutter.
|
||||
@ -421,7 +427,7 @@ impl InvokeUiSession for FlutterHandler {
|
||||
}
|
||||
|
||||
#[inline]
|
||||
#[cfg(not(any(target_os = "android", target_os = "ios")))]
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
fn on_rgba(&self, data: &mut Vec<u8>) {
|
||||
self.renderer.read().unwrap().on_rgba(data);
|
||||
if *self.notify_rendered.read().unwrap() {
|
||||
|
@ -1306,6 +1306,17 @@ pub fn main_hide_docker() -> SyncReturn<bool> {
|
||||
SyncReturn(true)
|
||||
}
|
||||
|
||||
pub fn main_use_texture_render() -> SyncReturn<bool> {
|
||||
#[cfg(not(feature = "flutter_texture_render"))]
|
||||
{
|
||||
SyncReturn(false)
|
||||
}
|
||||
#[cfg(feature = "flutter_texture_render")]
|
||||
{
|
||||
SyncReturn(true)
|
||||
}
|
||||
}
|
||||
|
||||
pub fn cm_start_listen_ipc_thread() {
|
||||
#[cfg(not(any(target_os = "android", target_os = "ios")))]
|
||||
crate::flutter::connection_manager::start_listen_ipc_thread();
|
||||
|
Loading…
x
Reference in New Issue
Block a user