diff --git a/Cargo.lock b/Cargo.lock index 1133105ce..c4fd31ea7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2083,6 +2083,15 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fon" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad46a0e6c9bc688823a742aa969b5c08fdc56c2a436ee00d5c6fbcb5982c55c4" +dependencies = [ + "libm", +] + [[package]] name = "foreign-types" version = "0.3.2" @@ -3293,6 +3302,12 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "libm" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "348108ab3fba42ec82ff6e9564fc4ca0247bdccdc68dd8af9764bbc79c3c8ffb" + [[package]] name = "libpulse-binding" version = "2.27.1" @@ -5124,6 +5139,7 @@ dependencies = [ "evdev", "flutter_rust_bridge", "flutter_rust_bridge_codegen", + "fon", "fruitbasket", "hbb_common", "hex", diff --git a/Cargo.toml b/Cargo.toml index 784b83537..49e2b66fc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -71,6 +71,7 @@ reqwest = { version = "0.11", features = ["blocking", "json", "rustls-tls"], def chrono = "0.4" cidr-utils = "0.5" libloading = "0.7" +fon = "0.6" [target.'cfg(not(any(target_os = "android", target_os = "linux")))'.dependencies] cpal = "0.15" diff --git a/src/client.rs b/src/client.rs index f37719641..f13823488 100644 --- a/src/client.rs +++ b/src/client.rs @@ -754,6 +754,7 @@ pub struct AudioHandler { #[cfg(not(any(target_os = "android", target_os = "linux")))] audio_stream: Option>, channels: u16, + device_channel: u16, #[cfg(not(any(target_os = "android", target_os = "linux")))] ready: Arc>, } @@ -826,22 +827,33 @@ impl AudioHandler { let sample_format = config.sample_format(); log::info!("Default output format: {:?}", config); log::info!("Remote input format: {:?}", format0); - let mut config: StreamConfig = config.into(); - config.channels = format0.channels as _; - match sample_format { - cpal::SampleFormat::I8 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::I16 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::I32 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::I64 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::U8 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::U16 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::U32 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::U64 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::F32 => self.build_output_stream::(&config, &device)?, - cpal::SampleFormat::F64 => self.build_output_stream::(&config, &device)?, - f => bail!("unsupported audio format: {:?}", f), - } + let config: StreamConfig = config.into(); self.sample_rate = (format0.sample_rate, config.sample_rate.0); + let mut build_output_stream = |config: StreamConfig| match sample_format { + cpal::SampleFormat::I8 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::I16 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::I32 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::I64 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::U8 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::U16 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::U32 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::U64 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::F32 => self.build_output_stream::(&config, &device), + cpal::SampleFormat::F64 => self.build_output_stream::(&config, &device), + f => bail!("unsupported audio format: {:?}", f), + }; + if config.channels > format0.channels as _ { + let no_rechannel_config = StreamConfig { + channels: format0.channels as _, + ..config.clone() + }; + if let Err(_) = build_output_stream(no_rechannel_config) { + build_output_stream(config)?; + } + } else { + build_output_stream(config)?; + } + Ok(()) } @@ -885,20 +897,25 @@ impl AudioHandler { let sample_rate0 = self.sample_rate.0; let sample_rate = self.sample_rate.1; let audio_buffer = self.audio_buffer.0.clone(); + let mut buffer = buffer[0..n].to_owned(); if sample_rate != sample_rate0 { - let buffer = crate::resample_channels( + buffer = crate::audio_resample( &buffer[0..n], sample_rate0, sample_rate, channels, ); - audio_buffer.lock().unwrap().push_slice_overwrite(&buffer); - } else { - audio_buffer - .lock() - .unwrap() - .push_slice_overwrite(&buffer[0..n]); } + if self.channels != self.device_channel { + buffer = crate::audio_rechannel( + buffer, + sample_rate, + sample_rate, + self.channels, + self.device_channel, + ); + } + audio_buffer.lock().unwrap().push_slice_overwrite(&buffer); } #[cfg(target_os = "android")] { @@ -921,6 +938,7 @@ impl AudioHandler { config: &StreamConfig, device: &Device, ) -> ResultType<()> { + self.device_channel = config.channels; let err_fn = move |err| { // too many errors, will improve later log::trace!("an error occurred on stream: {}", err); diff --git a/src/common.rs b/src/common.rs index d66938cf7..c2f02aa5a 100644 --- a/src/common.rs +++ b/src/common.rs @@ -273,7 +273,7 @@ pub fn resample_channels( } #[cfg(feature = "use_dasp")] -pub fn resample_channels( +pub fn audio_resample( data: &[f32], sample_rate0: u32, sample_rate: u32, @@ -309,7 +309,7 @@ pub fn resample_channels( } #[cfg(feature = "use_samplerate")] -pub fn resample_channels( +pub fn audio_resample( data: &[f32], sample_rate0: u32, sample_rate: u32, @@ -326,6 +326,158 @@ pub fn resample_channels( .unwrap_or_default() } +pub fn audio_rechannel( + input: Vec, + in_hz: u32, + out_hz: u32, + in_chan: u16, + output_chan: u16, +) -> Vec { + if in_chan == output_chan { + return input; + } + let mut input = input; + input.truncate(input.len() / in_chan as usize * in_chan as usize); + match (in_chan, output_chan) { + (1, 2) => audio_rechannel_1_2(&input, in_hz, out_hz), + (1, 3) => audio_rechannel_1_3(&input, in_hz, out_hz), + (1, 4) => audio_rechannel_1_4(&input, in_hz, out_hz), + (1, 5) => audio_rechannel_1_5(&input, in_hz, out_hz), + (1, 6) => audio_rechannel_1_6(&input, in_hz, out_hz), + (1, 7) => audio_rechannel_1_7(&input, in_hz, out_hz), + (1, 8) => audio_rechannel_1_8(&input, in_hz, out_hz), + (2, 1) => audio_rechannel_2_1(&input, in_hz, out_hz), + (2, 3) => audio_rechannel_2_3(&input, in_hz, out_hz), + (2, 4) => audio_rechannel_2_4(&input, in_hz, out_hz), + (2, 5) => audio_rechannel_2_5(&input, in_hz, out_hz), + (2, 6) => audio_rechannel_2_6(&input, in_hz, out_hz), + (2, 7) => audio_rechannel_2_7(&input, in_hz, out_hz), + (2, 8) => audio_rechannel_2_8(&input, in_hz, out_hz), + (3, 1) => audio_rechannel_3_1(&input, in_hz, out_hz), + (3, 2) => audio_rechannel_3_2(&input, in_hz, out_hz), + (3, 4) => audio_rechannel_3_4(&input, in_hz, out_hz), + (3, 5) => audio_rechannel_3_5(&input, in_hz, out_hz), + (3, 6) => audio_rechannel_3_6(&input, in_hz, out_hz), + (3, 7) => audio_rechannel_3_7(&input, in_hz, out_hz), + (3, 8) => audio_rechannel_3_8(&input, in_hz, out_hz), + (4, 1) => audio_rechannel_4_1(&input, in_hz, out_hz), + (4, 2) => audio_rechannel_4_2(&input, in_hz, out_hz), + (4, 3) => audio_rechannel_4_3(&input, in_hz, out_hz), + (4, 5) => audio_rechannel_4_5(&input, in_hz, out_hz), + (4, 6) => audio_rechannel_4_6(&input, in_hz, out_hz), + (4, 7) => audio_rechannel_4_7(&input, in_hz, out_hz), + (4, 8) => audio_rechannel_4_8(&input, in_hz, out_hz), + (5, 1) => audio_rechannel_5_1(&input, in_hz, out_hz), + (5, 2) => audio_rechannel_5_2(&input, in_hz, out_hz), + (5, 3) => audio_rechannel_5_3(&input, in_hz, out_hz), + (5, 4) => audio_rechannel_5_4(&input, in_hz, out_hz), + (5, 6) => audio_rechannel_5_6(&input, in_hz, out_hz), + (5, 7) => audio_rechannel_5_7(&input, in_hz, out_hz), + (5, 8) => audio_rechannel_5_8(&input, in_hz, out_hz), + (6, 1) => audio_rechannel_6_1(&input, in_hz, out_hz), + (6, 2) => audio_rechannel_6_2(&input, in_hz, out_hz), + (6, 3) => audio_rechannel_6_3(&input, in_hz, out_hz), + (6, 4) => audio_rechannel_6_4(&input, in_hz, out_hz), + (6, 5) => audio_rechannel_6_5(&input, in_hz, out_hz), + (6, 7) => audio_rechannel_6_7(&input, in_hz, out_hz), + (6, 8) => audio_rechannel_6_8(&input, in_hz, out_hz), + (7, 1) => audio_rechannel_7_1(&input, in_hz, out_hz), + (7, 2) => audio_rechannel_7_2(&input, in_hz, out_hz), + (7, 3) => audio_rechannel_7_3(&input, in_hz, out_hz), + (7, 4) => audio_rechannel_7_4(&input, in_hz, out_hz), + (7, 5) => audio_rechannel_7_5(&input, in_hz, out_hz), + (7, 6) => audio_rechannel_7_6(&input, in_hz, out_hz), + (7, 8) => audio_rechannel_7_8(&input, in_hz, out_hz), + (8, 1) => audio_rechannel_8_1(&input, in_hz, out_hz), + (8, 2) => audio_rechannel_8_2(&input, in_hz, out_hz), + (8, 3) => audio_rechannel_8_3(&input, in_hz, out_hz), + (8, 4) => audio_rechannel_8_4(&input, in_hz, out_hz), + (8, 5) => audio_rechannel_8_5(&input, in_hz, out_hz), + (8, 6) => audio_rechannel_8_6(&input, in_hz, out_hz), + (8, 7) => audio_rechannel_8_7(&input, in_hz, out_hz), + _ => input, + } +} + +macro_rules! audio_rechannel { + ($name:ident, $in_channels:expr, $out_channels:expr) => { + fn $name(input: &[f32], in_hz: u32, out_hz: u32) -> Vec { + use fon::{chan::Ch32, Audio, Frame}; + let mut in_audio = + Audio::::with_silence(in_hz, input.len() / $in_channels); + for (x, y) in input.chunks_exact($in_channels).zip(in_audio.iter_mut()) { + let mut f = Frame::::default(); + let mut i = 0; + for c in f.channels_mut() { + *c = x[i].into(); + i += 1; + } + *y = f; + } + Audio::::with_audio(out_hz, &in_audio) + .as_f32_slice() + .to_owned() + } + }; +} + +audio_rechannel!(audio_rechannel_1_2, 1, 2); +audio_rechannel!(audio_rechannel_1_3, 1, 3); +audio_rechannel!(audio_rechannel_1_4, 1, 4); +audio_rechannel!(audio_rechannel_1_5, 1, 5); +audio_rechannel!(audio_rechannel_1_6, 1, 6); +audio_rechannel!(audio_rechannel_1_7, 1, 7); +audio_rechannel!(audio_rechannel_1_8, 1, 8); +audio_rechannel!(audio_rechannel_2_1, 2, 1); +audio_rechannel!(audio_rechannel_2_3, 2, 3); +audio_rechannel!(audio_rechannel_2_4, 2, 4); +audio_rechannel!(audio_rechannel_2_5, 2, 5); +audio_rechannel!(audio_rechannel_2_6, 2, 6); +audio_rechannel!(audio_rechannel_2_7, 2, 7); +audio_rechannel!(audio_rechannel_2_8, 2, 8); +audio_rechannel!(audio_rechannel_3_1, 3, 1); +audio_rechannel!(audio_rechannel_3_2, 3, 2); +audio_rechannel!(audio_rechannel_3_4, 3, 4); +audio_rechannel!(audio_rechannel_3_5, 3, 5); +audio_rechannel!(audio_rechannel_3_6, 3, 6); +audio_rechannel!(audio_rechannel_3_7, 3, 7); +audio_rechannel!(audio_rechannel_3_8, 3, 8); +audio_rechannel!(audio_rechannel_4_1, 4, 1); +audio_rechannel!(audio_rechannel_4_2, 4, 2); +audio_rechannel!(audio_rechannel_4_3, 4, 3); +audio_rechannel!(audio_rechannel_4_5, 4, 5); +audio_rechannel!(audio_rechannel_4_6, 4, 6); +audio_rechannel!(audio_rechannel_4_7, 4, 7); +audio_rechannel!(audio_rechannel_4_8, 4, 8); +audio_rechannel!(audio_rechannel_5_1, 5, 1); +audio_rechannel!(audio_rechannel_5_2, 5, 2); +audio_rechannel!(audio_rechannel_5_3, 5, 3); +audio_rechannel!(audio_rechannel_5_4, 5, 4); +audio_rechannel!(audio_rechannel_5_6, 5, 6); +audio_rechannel!(audio_rechannel_5_7, 5, 7); +audio_rechannel!(audio_rechannel_5_8, 5, 8); +audio_rechannel!(audio_rechannel_6_1, 6, 1); +audio_rechannel!(audio_rechannel_6_2, 6, 2); +audio_rechannel!(audio_rechannel_6_3, 6, 3); +audio_rechannel!(audio_rechannel_6_4, 6, 4); +audio_rechannel!(audio_rechannel_6_5, 6, 5); +audio_rechannel!(audio_rechannel_6_7, 6, 7); +audio_rechannel!(audio_rechannel_6_8, 6, 8); +audio_rechannel!(audio_rechannel_7_1, 7, 1); +audio_rechannel!(audio_rechannel_7_2, 7, 2); +audio_rechannel!(audio_rechannel_7_3, 7, 3); +audio_rechannel!(audio_rechannel_7_4, 7, 4); +audio_rechannel!(audio_rechannel_7_5, 7, 5); +audio_rechannel!(audio_rechannel_7_6, 7, 6); +audio_rechannel!(audio_rechannel_7_8, 7, 8); +audio_rechannel!(audio_rechannel_8_1, 8, 1); +audio_rechannel!(audio_rechannel_8_2, 8, 2); +audio_rechannel!(audio_rechannel_8_3, 8, 3); +audio_rechannel!(audio_rechannel_8_4, 8, 4); +audio_rechannel!(audio_rechannel_8_5, 8, 5); +audio_rechannel!(audio_rechannel_8_6, 8, 6); +audio_rechannel!(audio_rechannel_8_7, 8, 7); + pub fn test_nat_type() { let mut i = 0; std::thread::spawn(move || loop { diff --git a/src/server/audio_service.rs b/src/server/audio_service.rs index c2b1d80f9..b30328479 100644 --- a/src/server/audio_service.rs +++ b/src/server/audio_service.rs @@ -140,21 +140,28 @@ mod cpal_impl { } fn send( - data: &[f32], + data: Vec, sample_rate0: u32, sample_rate: u32, - channels: u16, + device_channel: u16, + encode_channel: u16, encoder: &mut Encoder, sp: &GenericService, ) { - let buffer; - let data = if sample_rate0 != sample_rate { - buffer = crate::common::resample_channels(data, sample_rate0, sample_rate, channels); - &buffer - } else { - data - }; - send_f32(data, encoder, sp); + let mut data = data; + if sample_rate0 != sample_rate { + data = crate::common::audio_resample(&data, sample_rate0, sample_rate, device_channel); + } + if device_channel != encode_channel { + data = crate::common::audio_rechannel( + data, + sample_rate, + sample_rate, + device_channel, + encode_channel, + ) + } + send_f32(&data, encoder, sp); } #[cfg(windows)] @@ -230,23 +237,24 @@ mod cpal_impl { } else { 48000 }; + let ch = if config.channels() > 1 { Stereo } else { Mono }; let stream = match config.sample_format() { - I8 => build_input_stream::(device, &config, sp, sample_rate)?, - I16 => build_input_stream::(device, &config, sp, sample_rate)?, - I32 => build_input_stream::(device, &config, sp, sample_rate)?, - I64 => build_input_stream::(device, &config, sp, sample_rate)?, - U8 => build_input_stream::(device, &config, sp, sample_rate)?, - U16 => build_input_stream::(device, &config, sp, sample_rate)?, - U32 => build_input_stream::(device, &config, sp, sample_rate)?, - U64 => build_input_stream::(device, &config, sp, sample_rate)?, - F32 => build_input_stream::(device, &config, sp, sample_rate)?, - F64 => build_input_stream::(device, &config, sp, sample_rate)?, + I8 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + I16 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + I32 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + I64 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + U8 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + U16 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + U32 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + U64 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + F32 => build_input_stream::(device, &config, sp, sample_rate, ch)?, + F64 => build_input_stream::(device, &config, sp, sample_rate, ch)?, f => bail!("unsupported audio format: {:?}", f), }; stream.play()?; Ok(( Box::new(stream), - Arc::new(create_format_msg(sample_rate, config.channels())), + Arc::new(create_format_msg(sample_rate, ch as _)), )) } @@ -255,6 +263,7 @@ mod cpal_impl { config: &cpal::SupportedStreamConfig, sp: GenericService, sample_rate: u32, + encode_channel: magnum_opus::Channels, ) -> ResultType where T: cpal::SizedSample + dasp::sample::ToSample, @@ -268,19 +277,17 @@ mod cpal_impl { unsafe { AUDIO_ZERO_COUNT = 0; } - let mut encoder = Encoder::new( - sample_rate, - if config.channels() > 1 { Stereo } else { Mono }, - LowDelay, - )?; - let channels = config.channels(); + let device_channel = config.channels(); + let mut encoder = Encoder::new(sample_rate, encode_channel, LowDelay)?; // https://www.opus-codec.org/docs/html_api/group__opusencoder.html#gace941e4ef26ed844879fde342ffbe546 // https://chromium.googlesource.com/chromium/deps/opus/+/1.1.1/include/opus.h - let encode_len = sample_rate as usize * channels as usize / 100; // 10 ms + let frame_size = sample_rate as usize / 100; // 10 ms + let encode_len = frame_size * encode_channel as usize; + let rechannel_len = encode_len * device_channel as usize / encode_channel as usize; INPUT_BUFFER.lock().unwrap().clear(); let timeout = None; let stream_config = StreamConfig { - channels, + channels: device_channel, sample_rate: config.sample_rate(), buffer_size: BufferSize::Default, }; @@ -290,13 +297,14 @@ mod cpal_impl { let buffer: Vec = data.iter().map(|s| T::to_sample(*s)).collect(); let mut lock = INPUT_BUFFER.lock().unwrap(); lock.extend(buffer); - while lock.len() >= encode_len { - let frame: Vec = lock.drain(0..encode_len).collect(); + while lock.len() >= rechannel_len { + let frame: Vec = lock.drain(0..rechannel_len).collect(); send( - &frame, + frame, sample_rate_0, sample_rate, - channels, + device_channel, + encode_channel as _, &mut encoder, &sp, );