video handler holds only one decoder of the current codec format (#6939)
1. For example: when receiving h264 video frames, only 1 decoder is created, vram > ram 2. For creation and decoding failed: * Remove real_supported_decodings, this will update real existing decoders, replace it with the "mark_unsupported" vector. After creating the decoder failure, marks the codec as unsupported and updates supported decoding to the controlled side * Add `fail_counter` in the decoder. When decoding 10 consecutive frames failed, adding codec type to 'mark_unsupported' vector * The controlled end always ignores the unavailability of VP9 Signed-off-by: 21pages <pages21@163.com>
This commit is contained in:
21pages
GitHub
parent
2e16a2be56
commit
71d7398ae7
@ -240,7 +240,10 @@ fn test_av1(
|
||||
#[cfg(feature = "hwcodec")]
|
||||
mod hw {
|
||||
use hwcodec::ffmpeg::CodecInfo;
|
||||
use scrap::hwcodec::{HwDecoder, HwEncoder, HwEncoderConfig};
|
||||
use scrap::{
|
||||
hwcodec::{HwDecoder, HwEncoder, HwEncoderConfig},
|
||||
CodecFormat,
|
||||
};
|
||||
|
||||
use super::*;
|
||||
|
||||
@ -254,13 +257,8 @@ mod hw {
|
||||
if let Some(info) = best.h265 {
|
||||
test_encoder(width, height, quality, info, c, yuv_count, &mut h265s);
|
||||
}
|
||||
let best = HwDecoder::best();
|
||||
if let Some(info) = best.h264 {
|
||||
test_decoder(info, &h264s);
|
||||
}
|
||||
if let Some(info) = best.h265 {
|
||||
test_decoder(info, &h265s);
|
||||
}
|
||||
test_decoder(CodecFormat::H264, &h264s);
|
||||
test_decoder(CodecFormat::H265, &h265s);
|
||||
}
|
||||
|
||||
fn test_encoder(
|
||||
@ -322,16 +320,21 @@ mod hw {
|
||||
);
|
||||
}
|
||||
|
||||
fn test_decoder(info: CodecInfo, h26xs: &Vec<Vec<u8>>) {
|
||||
let mut decoder = HwDecoder::new(info.clone()).unwrap();
|
||||
fn test_decoder(format: CodecFormat, h26xs: &Vec<Vec<u8>>) {
|
||||
let mut decoder = HwDecoder::new(format).unwrap();
|
||||
let start = Instant::now();
|
||||
let mut cnt = 0;
|
||||
for h26x in h26xs {
|
||||
let _ = decoder.decode(h26x).unwrap();
|
||||
cnt += 1;
|
||||
}
|
||||
let device = format!("{:?}", info.hwdevice).to_lowercase();
|
||||
let device = format!("{:?}", decoder.info.hwdevice).to_lowercase();
|
||||
let device = device.split("_").last().unwrap();
|
||||
println!("{} {}: {:?}", info.name, device, start.elapsed() / cnt);
|
||||
println!(
|
||||
"{} {}: {:?}",
|
||||
decoder.info.name,
|
||||
device,
|
||||
start.elapsed() / cnt
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@ -10,14 +10,12 @@ use crate::gpucodec::*;
|
||||
#[cfg(feature = "hwcodec")]
|
||||
use crate::hwcodec::*;
|
||||
#[cfg(feature = "mediacodec")]
|
||||
use crate::mediacodec::{
|
||||
MediaCodecDecoder, MediaCodecDecoders, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT,
|
||||
};
|
||||
use crate::mediacodec::{MediaCodecDecoder, H264_DECODER_SUPPORT, H265_DECODER_SUPPORT};
|
||||
use crate::{
|
||||
aom::{self, AomDecoder, AomEncoder, AomEncoderConfig},
|
||||
common::GoogleImage,
|
||||
vpxcodec::{self, VpxDecoder, VpxDecoderConfig, VpxEncoder, VpxEncoderConfig, VpxVideoCodecId},
|
||||
CodecName, EncodeInput, EncodeYuvFormat, ImageRgb,
|
||||
CodecFormat, CodecName, EncodeInput, EncodeYuvFormat, ImageRgb,
|
||||
};
|
||||
|
||||
use hbb_common::{
|
||||
@ -96,13 +94,21 @@ pub struct Decoder {
|
||||
vp9: Option<VpxDecoder>,
|
||||
av1: Option<AomDecoder>,
|
||||
#[cfg(feature = "hwcodec")]
|
||||
hw: HwDecoders,
|
||||
h264_ram: Option<HwDecoder>,
|
||||
#[cfg(feature = "hwcodec")]
|
||||
h265_ram: Option<HwDecoder>,
|
||||
#[cfg(feature = "gpucodec")]
|
||||
gpu: GpuDecoders,
|
||||
h264_vram: Option<GpuDecoder>,
|
||||
#[cfg(feature = "gpucodec")]
|
||||
h265_vram: Option<GpuDecoder>,
|
||||
#[cfg(feature = "mediacodec")]
|
||||
h264_media_codec: MediaCodecDecoder,
|
||||
#[cfg(feature = "mediacodec")]
|
||||
h265_media_codec: MediaCodecDecoder,
|
||||
format: CodecFormat,
|
||||
valid: bool,
|
||||
#[cfg(feature = "hwcodec")]
|
||||
i420: Vec<u8>,
|
||||
#[cfg(feature = "mediacodec")]
|
||||
media_codec: MediaCodecDecoders,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
@ -372,6 +378,7 @@ impl Decoder {
|
||||
id_for_perfer: Option<&str>,
|
||||
_flutter: bool,
|
||||
_luid: Option<i64>,
|
||||
mark_unsupported: &Vec<CodecFormat>,
|
||||
) -> SupportedDecoding {
|
||||
let (prefer, prefer_chroma) = Self::preference(id_for_perfer);
|
||||
|
||||
@ -398,12 +405,12 @@ impl Decoder {
|
||||
}
|
||||
#[cfg(feature = "gpucodec")]
|
||||
if enable_gpucodec_option() && _flutter {
|
||||
decoding.ability_h264 |= if GpuDecoder::available(CodecName::H264GPU, _luid).len() > 0 {
|
||||
decoding.ability_h264 |= if GpuDecoder::available(CodecFormat::H264, _luid).len() > 0 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
};
|
||||
decoding.ability_h265 |= if GpuDecoder::available(CodecName::H265GPU, _luid).len() > 0 {
|
||||
decoding.ability_h265 |= if GpuDecoder::available(CodecFormat::H265, _luid).len() > 0 {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
@ -424,72 +431,148 @@ impl Decoder {
|
||||
0
|
||||
};
|
||||
}
|
||||
for unsupported in mark_unsupported {
|
||||
match unsupported {
|
||||
CodecFormat::VP8 => decoding.ability_vp8 = 0,
|
||||
CodecFormat::VP9 => decoding.ability_vp9 = 0,
|
||||
CodecFormat::AV1 => decoding.ability_av1 = 0,
|
||||
CodecFormat::H264 => decoding.ability_h264 = 0,
|
||||
CodecFormat::H265 => decoding.ability_h265 = 0,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
decoding
|
||||
}
|
||||
|
||||
pub fn exist_codecs(&self, _flutter: bool) -> CodecAbility {
|
||||
#[allow(unused_mut)]
|
||||
let mut ability = CodecAbility {
|
||||
vp8: self.vp8.is_some(),
|
||||
vp9: self.vp9.is_some(),
|
||||
av1: self.av1.is_some(),
|
||||
..Default::default()
|
||||
};
|
||||
pub fn new(format: CodecFormat, _luid: Option<i64>) -> Decoder {
|
||||
log::info!("try create new decoder, format: {format:?}, _luid: {_luid:?}");
|
||||
let (mut vp8, mut vp9, mut av1) = (None, None, None);
|
||||
#[cfg(feature = "hwcodec")]
|
||||
{
|
||||
ability.h264 |= self.hw.h264.is_some();
|
||||
ability.h265 |= self.hw.h265.is_some();
|
||||
}
|
||||
let (mut h264_ram, mut h265_ram) = (None, None);
|
||||
#[cfg(feature = "gpucodec")]
|
||||
if _flutter {
|
||||
ability.h264 |= self.gpu.h264.is_some();
|
||||
ability.h265 |= self.gpu.h265.is_some();
|
||||
}
|
||||
let (mut h264_vram, mut h265_vram) = (None, None);
|
||||
#[cfg(feature = "mediacodec")]
|
||||
{
|
||||
ability.h264 = self.media_codec.h264.is_some();
|
||||
ability.h265 = self.media_codec.h265.is_some();
|
||||
}
|
||||
ability
|
||||
}
|
||||
let (mut h264_media_codec, mut h265_media_codec) = (None, None);
|
||||
let mut valid = false;
|
||||
|
||||
pub fn new(_luid: Option<i64>) -> Decoder {
|
||||
let vp8 = VpxDecoder::new(VpxDecoderConfig {
|
||||
codec: VpxVideoCodecId::VP8,
|
||||
})
|
||||
.ok();
|
||||
let vp9 = VpxDecoder::new(VpxDecoderConfig {
|
||||
codec: VpxVideoCodecId::VP9,
|
||||
})
|
||||
.ok();
|
||||
let av1 = AomDecoder::new().ok();
|
||||
match format {
|
||||
CodecFormat::VP8 => {
|
||||
match VpxDecoder::new(VpxDecoderConfig {
|
||||
codec: VpxVideoCodecId::VP8,
|
||||
}) {
|
||||
Ok(v) => vp8 = Some(v),
|
||||
Err(e) => log::error!("create VP8 decoder failed: {}", e),
|
||||
}
|
||||
valid = vp8.is_some();
|
||||
}
|
||||
CodecFormat::VP9 => {
|
||||
match VpxDecoder::new(VpxDecoderConfig {
|
||||
codec: VpxVideoCodecId::VP9,
|
||||
}) {
|
||||
Ok(v) => vp9 = Some(v),
|
||||
Err(e) => log::error!("create VP9 decoder failed: {}", e),
|
||||
}
|
||||
valid = vp9.is_some();
|
||||
}
|
||||
CodecFormat::AV1 => {
|
||||
match AomDecoder::new() {
|
||||
Ok(v) => av1 = Some(v),
|
||||
Err(e) => log::error!("create AV1 decoder failed: {}", e),
|
||||
}
|
||||
valid = av1.is_some();
|
||||
}
|
||||
CodecFormat::H264 => {
|
||||
#[cfg(feature = "gpucodec")]
|
||||
if !valid && enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
|
||||
match GpuDecoder::new(format, _luid) {
|
||||
Ok(v) => h264_vram = Some(v),
|
||||
Err(e) => log::error!("create H264 vram decoder failed: {}", e),
|
||||
}
|
||||
valid = h264_vram.is_some();
|
||||
}
|
||||
#[cfg(feature = "hwcodec")]
|
||||
if !valid && enable_hwcodec_option() {
|
||||
match HwDecoder::new(format) {
|
||||
Ok(v) => h264_ram = Some(v),
|
||||
Err(e) => log::error!("create H264 ram decoder failed: {}", e),
|
||||
}
|
||||
valid = h264_ram.is_some();
|
||||
}
|
||||
#[cfg(feature = "mediacodec")]
|
||||
if !valid && enable_hwcodec_option() {
|
||||
h264_media_codec = MediaCodecDecoder::new(format);
|
||||
if h264_media_codec.is_none() {
|
||||
log::error!("create H264 media codec decoder failed");
|
||||
}
|
||||
valid = h264_media_codec.is_some();
|
||||
}
|
||||
}
|
||||
CodecFormat::H265 => {
|
||||
#[cfg(feature = "gpucodec")]
|
||||
if !valid && enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
|
||||
match GpuDecoder::new(format, _luid) {
|
||||
Ok(v) => h265_vram = Some(v),
|
||||
Err(e) => log::error!("create H265 vram decoder failed: {}", e),
|
||||
}
|
||||
valid = h265_vram.is_some();
|
||||
}
|
||||
#[cfg(feature = "hwcodec")]
|
||||
if !valid && enable_hwcodec_option() {
|
||||
match HwDecoder::new(format) {
|
||||
Ok(v) => h265_ram = Some(v),
|
||||
Err(e) => log::error!("create H265 ram decoder failed: {}", e),
|
||||
}
|
||||
valid = h265_ram.is_some();
|
||||
}
|
||||
#[cfg(feature = "mediacodec")]
|
||||
if !valid && enable_hwcodec_option() {
|
||||
h265_media_codec = MediaCodecDecoder::new(format);
|
||||
if h265_media_codec.is_none() {
|
||||
log::error!("create H265 media codec decoder failed");
|
||||
}
|
||||
valid = h265_media_codec.is_some();
|
||||
}
|
||||
}
|
||||
CodecFormat::Unknown => {
|
||||
log::error!("unknown codec format, cannot create decoder");
|
||||
}
|
||||
}
|
||||
if !valid {
|
||||
log::error!("failed to create {format:?} decoder");
|
||||
} else {
|
||||
log::info!("create {format:?} decoder success");
|
||||
}
|
||||
Decoder {
|
||||
vp8,
|
||||
vp9,
|
||||
av1,
|
||||
#[cfg(feature = "hwcodec")]
|
||||
hw: if enable_hwcodec_option() {
|
||||
HwDecoder::new_decoders()
|
||||
} else {
|
||||
HwDecoders::default()
|
||||
},
|
||||
h264_ram,
|
||||
#[cfg(feature = "hwcodec")]
|
||||
h265_ram,
|
||||
#[cfg(feature = "gpucodec")]
|
||||
gpu: if enable_gpucodec_option() && _luid.clone().unwrap_or_default() != 0 {
|
||||
GpuDecoder::new_decoders(_luid)
|
||||
} else {
|
||||
GpuDecoders::default()
|
||||
},
|
||||
h264_vram,
|
||||
#[cfg(feature = "gpucodec")]
|
||||
h265_vram,
|
||||
#[cfg(feature = "mediacodec")]
|
||||
h264_media_codec,
|
||||
#[cfg(feature = "mediacodec")]
|
||||
h265_media_codec,
|
||||
format,
|
||||
valid,
|
||||
#[cfg(feature = "hwcodec")]
|
||||
i420: vec![],
|
||||
#[cfg(feature = "mediacodec")]
|
||||
media_codec: if enable_hwcodec_option() {
|
||||
MediaCodecDecoder::new_decoders()
|
||||
} else {
|
||||
MediaCodecDecoders::default()
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
pub fn format(&self) -> CodecFormat {
|
||||
self.format
|
||||
}
|
||||
|
||||
pub fn valid(&self) -> bool {
|
||||
self.valid
|
||||
}
|
||||
|
||||
// rgb [in/out] fmt and stride must be set in ImageRgb
|
||||
pub fn handle_video_frame(
|
||||
&mut self,
|
||||
@ -525,12 +608,12 @@ impl Decoder {
|
||||
video_frame::Union::H264s(h264s) => {
|
||||
*chroma = Some(Chroma::I420);
|
||||
#[cfg(feature = "gpucodec")]
|
||||
if let Some(decoder) = &mut self.gpu.h264 {
|
||||
if let Some(decoder) = &mut self.h264_vram {
|
||||
*_pixelbuffer = false;
|
||||
return Decoder::handle_gpu_video_frame(decoder, h264s, _texture);
|
||||
}
|
||||
#[cfg(feature = "hwcodec")]
|
||||
if let Some(decoder) = &mut self.hw.h264 {
|
||||
if let Some(decoder) = &mut self.h264_ram {
|
||||
return Decoder::handle_hw_video_frame(decoder, h264s, rgb, &mut self.i420);
|
||||
}
|
||||
Err(anyhow!("don't support h264!"))
|
||||
@ -539,12 +622,12 @@ impl Decoder {
|
||||
video_frame::Union::H265s(h265s) => {
|
||||
*chroma = Some(Chroma::I420);
|
||||
#[cfg(feature = "gpucodec")]
|
||||
if let Some(decoder) = &mut self.gpu.h265 {
|
||||
if let Some(decoder) = &mut self.h265_vram {
|
||||
*_pixelbuffer = false;
|
||||
return Decoder::handle_gpu_video_frame(decoder, h265s, _texture);
|
||||
}
|
||||
#[cfg(feature = "hwcodec")]
|
||||
if let Some(decoder) = &mut self.hw.h265 {
|
||||
if let Some(decoder) = &mut self.h265_ram {
|
||||
return Decoder::handle_hw_video_frame(decoder, h265s, rgb, &mut self.i420);
|
||||
}
|
||||
Err(anyhow!("don't support h265!"))
|
||||
@ -552,7 +635,7 @@ impl Decoder {
|
||||
#[cfg(feature = "mediacodec")]
|
||||
video_frame::Union::H264s(h264s) => {
|
||||
*chroma = Some(Chroma::I420);
|
||||
if let Some(decoder) = &mut self.media_codec.h264 {
|
||||
if let Some(decoder) = &mut self.h264_media_codec {
|
||||
Decoder::handle_mediacodec_video_frame(decoder, h264s, rgb)
|
||||
} else {
|
||||
Err(anyhow!("don't support h264!"))
|
||||
@ -561,7 +644,7 @@ impl Decoder {
|
||||
#[cfg(feature = "mediacodec")]
|
||||
video_frame::Union::H265s(h265s) => {
|
||||
*chroma = Some(Chroma::I420);
|
||||
if let Some(decoder) = &mut self.media_codec.h265 {
|
||||
if let Some(decoder) = &mut self.h265_media_codec {
|
||||
Decoder::handle_mediacodec_video_frame(decoder, h265s, rgb)
|
||||
} else {
|
||||
Err(anyhow!("don't support h265!"))
|
||||
|
||||
@ -6,7 +6,7 @@ use std::{
|
||||
|
||||
use crate::{
|
||||
codec::{base_bitrate, enable_gpucodec_option, EncoderApi, EncoderCfg, Quality},
|
||||
AdapterDevice, CodecName, EncodeInput, EncodeYuvFormat, Pixfmt,
|
||||
AdapterDevice, CodecFormat, CodecName, EncodeInput, EncodeYuvFormat, Pixfmt,
|
||||
};
|
||||
use gpucodec::gpu_common::{
|
||||
self, Available, DecodeContext, DynamicContext, EncodeContext, FeatureContext, MAX_GOP,
|
||||
@ -87,7 +87,10 @@ impl EncoderApi for GpuEncoder {
|
||||
last_frame_len: 0,
|
||||
same_bad_len_counter: 0,
|
||||
}),
|
||||
Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
|
||||
Err(_) => {
|
||||
hbb_common::config::GpucodecConfig::clear();
|
||||
Err(anyhow!(format!("Failed to create encoder")))
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err(anyhow!("encoder type mismatch")),
|
||||
@ -300,8 +303,8 @@ pub struct GpuDecoders {
|
||||
}
|
||||
|
||||
impl GpuDecoder {
|
||||
pub fn try_get(name: CodecName, luid: Option<i64>) -> Option<DecodeContext> {
|
||||
let v: Vec<_> = Self::available(name, luid);
|
||||
pub fn try_get(format: CodecFormat, luid: Option<i64>) -> Option<DecodeContext> {
|
||||
let v: Vec<_> = Self::available(format, luid);
|
||||
if v.len() > 0 {
|
||||
Some(v[0].clone())
|
||||
} else {
|
||||
@ -309,11 +312,11 @@ impl GpuDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn available(name: CodecName, luid: Option<i64>) -> Vec<DecodeContext> {
|
||||
pub fn available(format: CodecFormat, luid: Option<i64>) -> Vec<DecodeContext> {
|
||||
let luid = luid.unwrap_or_default();
|
||||
let data_format = match name {
|
||||
CodecName::H264GPU => gpu_common::DataFormat::H264,
|
||||
CodecName::H265GPU => gpu_common::DataFormat::H265,
|
||||
let data_format = match format {
|
||||
CodecFormat::H264 => gpu_common::DataFormat::H264,
|
||||
CodecFormat::H265 => gpu_common::DataFormat::H265,
|
||||
_ => return vec![],
|
||||
};
|
||||
get_available_config()
|
||||
@ -337,28 +340,18 @@ impl GpuDecoder {
|
||||
)
|
||||
}
|
||||
|
||||
pub fn new_decoders(luid: Option<i64>) -> GpuDecoders {
|
||||
let mut h264: Option<GpuDecoder> = None;
|
||||
let mut h265: Option<GpuDecoder> = None;
|
||||
if let Ok(decoder) = GpuDecoder::new(CodecName::H264GPU, luid) {
|
||||
h264 = Some(decoder);
|
||||
}
|
||||
if let Ok(decoder) = GpuDecoder::new(CodecName::H265GPU, luid) {
|
||||
h265 = Some(decoder);
|
||||
}
|
||||
log::info!(
|
||||
"new gpu decoders, support h264: {}, h265: {}",
|
||||
h264.is_some(),
|
||||
h265.is_some()
|
||||
);
|
||||
GpuDecoders { h264, h265 }
|
||||
}
|
||||
|
||||
pub fn new(name: CodecName, luid: Option<i64>) -> ResultType<Self> {
|
||||
let ctx = Self::try_get(name, luid).ok_or(anyhow!("Failed to get decode context"))?;
|
||||
pub fn new(format: CodecFormat, luid: Option<i64>) -> ResultType<Self> {
|
||||
log::info!("try create {format:?} vram decoder, luid: {luid:?}");
|
||||
let ctx = Self::try_get(format, luid).ok_or(anyhow!("Failed to get decode context"))?;
|
||||
match Decoder::new(ctx) {
|
||||
Ok(decoder) => Ok(Self { decoder }),
|
||||
Err(_) => Err(anyhow!(format!("Failed to create decoder"))),
|
||||
Err(_) => {
|
||||
hbb_common::config::GpucodecConfig::clear();
|
||||
Err(anyhow!(format!(
|
||||
"Failed to create decoder, format: {:?}",
|
||||
format
|
||||
)))
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<GpuDecoderImage>> {
|
||||
|
||||
@ -1,10 +1,10 @@
|
||||
use crate::{
|
||||
codec::{base_bitrate, codec_thread_num, EncoderApi, EncoderCfg, Quality as Q},
|
||||
hw, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
|
||||
hw, CodecFormat, EncodeInput, ImageFormat, ImageRgb, Pixfmt, HW_STRIDE_ALIGN,
|
||||
};
|
||||
use hbb_common::{
|
||||
allow_err,
|
||||
anyhow::{anyhow, Context},
|
||||
anyhow::{anyhow, bail, Context},
|
||||
bytes::Bytes,
|
||||
config::HwCodecConfig,
|
||||
log,
|
||||
@ -94,7 +94,10 @@ impl EncoderApi for HwEncoder {
|
||||
height: ctx.height as _,
|
||||
bitrate,
|
||||
}),
|
||||
Err(_) => Err(anyhow!(format!("Failed to create encoder"))),
|
||||
Err(_) => {
|
||||
HwCodecConfig::clear();
|
||||
Err(anyhow!(format!("Failed to create encoder")))
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => Err(anyhow!("encoder type mismatch")),
|
||||
@ -230,31 +233,26 @@ impl HwDecoder {
|
||||
})
|
||||
}
|
||||
|
||||
pub fn new_decoders() -> HwDecoders {
|
||||
pub fn new(format: CodecFormat) -> ResultType<Self> {
|
||||
log::info!("try create {format:?} ram decoder");
|
||||
let best = HwDecoder::best();
|
||||
let mut h264: Option<HwDecoder> = None;
|
||||
let mut h265: Option<HwDecoder> = None;
|
||||
let mut fail = false;
|
||||
|
||||
if let Some(info) = best.h264 {
|
||||
h264 = HwDecoder::new(info).ok();
|
||||
if h264.is_none() {
|
||||
fail = true;
|
||||
let info = match format {
|
||||
CodecFormat::H264 => {
|
||||
if let Some(info) = best.h264 {
|
||||
info
|
||||
} else {
|
||||
bail!("no h264 decoder, should not be here");
|
||||
}
|
||||
}
|
||||
}
|
||||
if let Some(info) = best.h265 {
|
||||
h265 = HwDecoder::new(info).ok();
|
||||
if h265.is_none() {
|
||||
fail = true;
|
||||
CodecFormat::H265 => {
|
||||
if let Some(info) = best.h265 {
|
||||
info
|
||||
} else {
|
||||
bail!("no h265 decoder, should not be here");
|
||||
}
|
||||
}
|
||||
}
|
||||
if fail {
|
||||
hwcodec_new_check_process();
|
||||
}
|
||||
HwDecoders { h264, h265 }
|
||||
}
|
||||
|
||||
pub fn new(info: CodecInfo) -> ResultType<Self> {
|
||||
_ => bail!("unsupported format: {:?}", format),
|
||||
};
|
||||
let ctx = DecodeContext {
|
||||
name: info.name.clone(),
|
||||
device_type: info.hwdevice.clone(),
|
||||
@ -262,7 +260,10 @@ impl HwDecoder {
|
||||
};
|
||||
match Decoder::new(ctx) {
|
||||
Ok(decoder) => Ok(HwDecoder { decoder, info }),
|
||||
Err(_) => Err(anyhow!(format!("Failed to create decoder"))),
|
||||
Err(_) => {
|
||||
HwCodecConfig::clear();
|
||||
Err(anyhow!(format!("Failed to create decoder")))
|
||||
}
|
||||
}
|
||||
}
|
||||
pub fn decode(&mut self, data: &[u8]) -> ResultType<Vec<HwDecoderImage>> {
|
||||
|
||||
@ -10,7 +10,7 @@ use std::{
|
||||
use crate::ImageFormat;
|
||||
use crate::{
|
||||
codec::{EncoderApi, EncoderCfg},
|
||||
I420ToABGR, I420ToARGB, ImageRgb,
|
||||
CodecFormat, I420ToABGR, I420ToARGB, ImageRgb,
|
||||
};
|
||||
|
||||
/// MediaCodec mime type name
|
||||
@ -37,17 +37,16 @@ impl Deref for MediaCodecDecoder {
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct MediaCodecDecoders {
|
||||
pub h264: Option<MediaCodecDecoder>,
|
||||
pub h265: Option<MediaCodecDecoder>,
|
||||
}
|
||||
|
||||
impl MediaCodecDecoder {
|
||||
pub fn new_decoders() -> MediaCodecDecoders {
|
||||
let h264 = create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder);
|
||||
let h265 = create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder);
|
||||
MediaCodecDecoders { h264, h265 }
|
||||
pub fn new(format: CodecFormat) -> Option<MediaCodecDecoder> {
|
||||
match format {
|
||||
CodecFormat::H264 => create_media_codec(H264_MIME_TYPE, MediaCodecDirection::Decoder),
|
||||
CodecFormat::H265 => create_media_codec(H265_MIME_TYPE, MediaCodecDirection::Decoder),
|
||||
_ => {
|
||||
log::error!("Unsupported codec format: {}", format);
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// rgb [in/out] fmt and stride must be set in ImageRgb
|
||||
|
||||
@ -251,7 +251,7 @@ pub enum CodecName {
|
||||
H265GPU,
|
||||
}
|
||||
|
||||
#[derive(PartialEq, Debug, Clone)]
|
||||
#[derive(PartialEq, Debug, Clone, Copy)]
|
||||
pub enum CodecFormat {
|
||||
VP8,
|
||||
VP9,
|
||||
|
||||
117
src/client.rs
117
src/client.rs
@ -51,7 +51,7 @@ pub use helper::*;
|
||||
use scrap::{
|
||||
codec::Decoder,
|
||||
record::{Recorder, RecorderContext},
|
||||
ImageFormat, ImageRgb,
|
||||
CodecFormat, ImageFormat, ImageRgb,
|
||||
};
|
||||
|
||||
use crate::{
|
||||
@ -76,6 +76,7 @@ pub mod io_loop;
|
||||
pub const MILLI1: Duration = Duration::from_millis(1);
|
||||
pub const SEC30: Duration = Duration::from_secs(30);
|
||||
pub const VIDEO_QUEUE_SIZE: usize = 120;
|
||||
const MAX_DECODE_FAIL_COUNTER: usize = 10; // Currently, failed decode cause refresh_video, so make it small
|
||||
|
||||
#[cfg(all(target_os = "linux", feature = "linux_headless"))]
|
||||
#[cfg(not(any(feature = "flatpak", feature = "appimage")))]
|
||||
@ -1027,24 +1028,25 @@ pub struct VideoHandler {
|
||||
recorder: Arc<Mutex<Option<Recorder>>>,
|
||||
record: bool,
|
||||
_display: usize, // useful for debug
|
||||
fail_counter: usize,
|
||||
}
|
||||
|
||||
impl VideoHandler {
|
||||
/// Create a new video handler.
|
||||
pub fn new(_display: usize) -> Self {
|
||||
pub fn new(format: CodecFormat, _display: usize) -> Self {
|
||||
#[cfg(all(feature = "gpucodec", feature = "flutter"))]
|
||||
let luid = crate::flutter::get_adapter_luid();
|
||||
#[cfg(not(all(feature = "gpucodec", feature = "flutter")))]
|
||||
let luid = Default::default();
|
||||
println!("new session_get_adapter_luid: {:?}", luid);
|
||||
log::info!("new video handler for display #{_display}");
|
||||
log::info!("new video handler for display #{_display}, format: {format:?}, luid: {luid:?}");
|
||||
VideoHandler {
|
||||
decoder: Decoder::new(luid),
|
||||
decoder: Decoder::new(format, luid),
|
||||
rgb: ImageRgb::new(ImageFormat::ARGB, crate::DST_STRIDE_RGBA),
|
||||
texture: std::ptr::null_mut(),
|
||||
recorder: Default::default(),
|
||||
record: false,
|
||||
_display,
|
||||
fail_counter: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@ -1056,6 +1058,10 @@ impl VideoHandler {
|
||||
pixelbuffer: &mut bool,
|
||||
chroma: &mut Option<Chroma>,
|
||||
) -> ResultType<bool> {
|
||||
let format = CodecFormat::from(&vf);
|
||||
if format != self.decoder.format() {
|
||||
self.reset(Some(format));
|
||||
}
|
||||
match &vf.union {
|
||||
Some(frame) => {
|
||||
let res = self.decoder.handle_video_frame(
|
||||
@ -1065,6 +1071,13 @@ impl VideoHandler {
|
||||
pixelbuffer,
|
||||
chroma,
|
||||
);
|
||||
if res.as_ref().is_ok_and(|x| *x) {
|
||||
self.fail_counter = 0;
|
||||
} else {
|
||||
if self.fail_counter < usize::MAX {
|
||||
self.fail_counter += 1
|
||||
}
|
||||
}
|
||||
if self.record {
|
||||
self.recorder
|
||||
.lock()
|
||||
@ -1078,13 +1091,15 @@ impl VideoHandler {
|
||||
}
|
||||
}
|
||||
|
||||
/// Reset the decoder.
|
||||
pub fn reset(&mut self) {
|
||||
/// Reset the decoder, change format if it is Some
|
||||
pub fn reset(&mut self, format: Option<CodecFormat>) {
|
||||
#[cfg(all(feature = "flutter", feature = "gpucodec"))]
|
||||
let luid = crate::flutter::get_adapter_luid();
|
||||
#[cfg(not(all(feature = "flutter", feature = "gpucodec")))]
|
||||
let luid = None;
|
||||
self.decoder = Decoder::new(luid);
|
||||
let format = format.unwrap_or(self.decoder.format());
|
||||
self.decoder = Decoder::new(format, luid);
|
||||
self.fail_counter = 0;
|
||||
}
|
||||
|
||||
/// Start or stop screen record.
|
||||
@ -1133,6 +1148,7 @@ pub struct LoginConfigHandler {
|
||||
pub other_server: Option<(String, String, String)>,
|
||||
pub custom_fps: Arc<Mutex<Option<usize>>>,
|
||||
pub adapter_luid: Option<i64>,
|
||||
pub mark_unsupported: Vec<CodecFormat>,
|
||||
}
|
||||
|
||||
impl Deref for LoginConfigHandler {
|
||||
@ -1562,6 +1578,7 @@ impl LoginConfigHandler {
|
||||
Some(&self.id),
|
||||
cfg!(feature = "flutter"),
|
||||
self.adapter_luid,
|
||||
&self.mark_unsupported,
|
||||
));
|
||||
n += 1;
|
||||
|
||||
@ -1947,6 +1964,7 @@ impl LoginConfigHandler {
|
||||
Some(&self.id),
|
||||
cfg!(feature = "flutter"),
|
||||
self.adapter_luid,
|
||||
&self.mark_unsupported,
|
||||
);
|
||||
let mut misc = Misc::new();
|
||||
misc.set_option(OptionMessage {
|
||||
@ -1958,44 +1976,6 @@ impl LoginConfigHandler {
|
||||
msg_out
|
||||
}
|
||||
|
||||
fn real_supported_decodings(
|
||||
&self,
|
||||
handler_controller_map: &Vec<VideoHandlerController>,
|
||||
) -> Data {
|
||||
let abilities: Vec<CodecAbility> = handler_controller_map
|
||||
.iter()
|
||||
.map(|h| h.handler.decoder.exist_codecs(cfg!(feature = "flutter")))
|
||||
.collect();
|
||||
let all = |ability: fn(&CodecAbility) -> bool| -> i32 {
|
||||
if abilities.iter().all(|d| ability(d)) {
|
||||
1
|
||||
} else {
|
||||
0
|
||||
}
|
||||
};
|
||||
let decoding = scrap::codec::Decoder::supported_decodings(
|
||||
Some(&self.id),
|
||||
cfg!(feature = "flutter"),
|
||||
self.adapter_luid,
|
||||
);
|
||||
let decoding = SupportedDecoding {
|
||||
ability_vp8: all(|e| e.vp8),
|
||||
ability_vp9: all(|e| e.vp9),
|
||||
ability_av1: all(|e| e.av1),
|
||||
ability_h264: all(|e| e.h264),
|
||||
ability_h265: all(|e| e.h265),
|
||||
..decoding
|
||||
};
|
||||
let mut misc = Misc::new();
|
||||
misc.set_option(OptionMessage {
|
||||
supported_decoding: hbb_common::protobuf::MessageField::some(decoding),
|
||||
..Default::default()
|
||||
});
|
||||
let mut msg_out = Message::new();
|
||||
msg_out.set_misc(misc);
|
||||
Data::Message(msg_out)
|
||||
}
|
||||
|
||||
pub fn restart_remote_device(&self) -> Message {
|
||||
let mut misc = Misc::new();
|
||||
misc.set_restart_remote_device(true);
|
||||
@ -2088,27 +2068,17 @@ where
|
||||
};
|
||||
let display = vf.display as usize;
|
||||
let start = std::time::Instant::now();
|
||||
let mut created_new_handler = false;
|
||||
let format = CodecFormat::from(&vf);
|
||||
if handler_controller_map.len() <= display {
|
||||
for _i in handler_controller_map.len()..=display {
|
||||
handler_controller_map.push(VideoHandlerController {
|
||||
handler: VideoHandler::new(_i),
|
||||
handler: VideoHandler::new(format, _i),
|
||||
count: 0,
|
||||
duration: std::time::Duration::ZERO,
|
||||
skip_beginning: 0,
|
||||
});
|
||||
created_new_handler = true;
|
||||
}
|
||||
}
|
||||
if created_new_handler {
|
||||
session.send(
|
||||
session
|
||||
.lc
|
||||
.read()
|
||||
.unwrap()
|
||||
.real_supported_decodings(&handler_controller_map),
|
||||
);
|
||||
}
|
||||
if let Some(handler_controller) = handler_controller_map.get_mut(display) {
|
||||
let mut pixelbuffer = true;
|
||||
let mut tmp_chroma = None;
|
||||
@ -2172,17 +2142,32 @@ where
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
// check invalid decoders
|
||||
let mut should_update_supported = false;
|
||||
handler_controller_map
|
||||
.iter()
|
||||
.map(|h| {
|
||||
if !h.handler.decoder.valid() || h.handler.fail_counter >= MAX_DECODE_FAIL_COUNTER {
|
||||
let mut lc = session.lc.write().unwrap();
|
||||
let format = h.handler.decoder.format();
|
||||
if !lc.mark_unsupported.contains(&format) {
|
||||
lc.mark_unsupported.push(format);
|
||||
should_update_supported = true;
|
||||
log::info!("mark {format:?} decoder as unsupported, valid:{}, fail_counter:{}, all unsupported:{:?}", h.handler.decoder.valid(), h.handler.fail_counter, lc.mark_unsupported);
|
||||
}
|
||||
}
|
||||
})
|
||||
.count();
|
||||
if should_update_supported {
|
||||
session.send(Data::Message(
|
||||
session.lc.read().unwrap().update_supported_decodings(),
|
||||
));
|
||||
}
|
||||
}
|
||||
MediaData::Reset(display) => {
|
||||
if let Some(handler_controler) = handler_controller_map.get_mut(display) {
|
||||
handler_controler.handler.reset();
|
||||
session.send(
|
||||
session
|
||||
.lc
|
||||
.read()
|
||||
.unwrap()
|
||||
.real_supported_decodings(&handler_controller_map),
|
||||
);
|
||||
handler_controler.handler.reset(None);
|
||||
}
|
||||
}
|
||||
MediaData::RecordScreen(start, display, w, h, id) => {
|
||||
|
||||
@ -846,7 +846,7 @@ pub fn has_gpucodec() -> bool {
|
||||
#[cfg(feature = "flutter")]
|
||||
#[inline]
|
||||
pub fn supported_hwdecodings() -> (bool, bool) {
|
||||
let decoding = scrap::codec::Decoder::supported_decodings(None, true, None);
|
||||
let decoding = scrap::codec::Decoder::supported_decodings(None, true, None, &vec![]);
|
||||
#[allow(unused_mut)]
|
||||
let (mut h264, mut h265) = (decoding.ability_h264 > 0, decoding.ability_h265 > 0);
|
||||
#[cfg(feature = "gpucodec")]
|
||||
|
||||
@ -437,8 +437,13 @@ impl<T: InvokeUiSession> Session<T> {
|
||||
|
||||
pub fn alternative_codecs(&self) -> (bool, bool, bool, bool) {
|
||||
let luid = self.lc.read().unwrap().adapter_luid;
|
||||
let decoder =
|
||||
scrap::codec::Decoder::supported_decodings(None, cfg!(feature = "flutter"), luid);
|
||||
let mark_unsupported = self.lc.read().unwrap().mark_unsupported.clone();
|
||||
let decoder = scrap::codec::Decoder::supported_decodings(
|
||||
None,
|
||||
cfg!(feature = "flutter"),
|
||||
luid,
|
||||
&mark_unsupported,
|
||||
);
|
||||
let mut vp8 = decoder.ability_vp8 > 0;
|
||||
let mut av1 = decoder.ability_av1 > 0;
|
||||
let mut h264 = decoder.ability_h264 > 0;
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user