diff --git a/apps/social/client/src/custom_audio/microphone.rs b/apps/social/client/src/custom_audio/microphone.rs index 4a82176..32abcfa 100644 --- a/apps/social/client/src/custom_audio/microphone.rs +++ b/apps/social/client/src/custom_audio/microphone.rs @@ -5,6 +5,7 @@ use bevy::prelude::{warn, Commands, Resource, Startup}; use cpal::traits::{DeviceTrait, HostTrait, StreamTrait}; use std::sync::mpsc::{channel, Receiver}; use std::sync::Mutex; +use bevy::prelude::info; pub struct MicrophonePlugin; @@ -33,7 +34,7 @@ impl Default for MicrophoneConfig { } pub fn create_microphone(mut commands: Commands) { - let microphone_config = MicrophoneConfig::default(); + let mut microphone_config = MicrophoneConfig::default(); // we wanna share the output from our thread loop thing in here continuously with the rest of bevy. let (tx, rx) = channel(); @@ -49,11 +50,24 @@ pub fn create_microphone(mut commands: Commands) { Ok(configs) => configs, Err(err) => return warn!("supported stream config error, microphone functionality will be disabled, error: {}", err), }; + for config in configs { + warn!("supported microphone config: {:#?}", config); + } + let mut configs = match device.supported_input_configs() { + Ok(configs) => configs, + Err(err) => return warn!("supported stream config error, microphone functionality will be disabled, error: {}", err), + }; + + #[cfg(target_os = "android")] + { + microphone_config.channels = 2; + } + let config = match configs .find(|c| { c.sample_format() == cpal::SampleFormat::F32 && c.channels() == microphone_config.channels - && c.min_sample_rate().0 < microphone_config.sample_rate - && c.max_sample_rate().0 > microphone_config.sample_rate + && c.min_sample_rate().0 <= microphone_config.sample_rate + && c.max_sample_rate().0 >= microphone_config.sample_rate }) { None => return warn!("microphone config of {:?} not supported, microphone functionality will be disabled", microphone_config), diff --git a/apps/social/client/src/voice_chat/mod.rs b/apps/social/client/src/voice_chat/mod.rs index 3996877..96d2c2a 100644 --- a/apps/social/client/src/voice_chat/mod.rs +++ b/apps/social/client/src/voice_chat/mod.rs @@ -12,14 +12,26 @@ pub struct VoiceChatPlugin; impl bevy::prelude::Plugin for VoiceChatPlugin { fn build(&self, app: &mut App) { - app.insert_non_send_resource(MicrophoneEncoder( - Encoder::new(48_000, Channels::Mono, Application::Voip) - .expect("unable to create microphone audio compressing encoder"), - )); - app.insert_non_send_resource(MicrophoneDecoder( - Decoder::new(48_000, Channels::Mono) + #[cfg(target_os = "android")] + { + app.insert_non_send_resource(crate::voice_chat::MicrophoneEncoder( + Encoder::new(48_000, Channels::Stereo, Application::Voip) + .expect("unable to create microphone audio compressing encoder"), + )); + } + #[cfg(not(target_os = "android"))] + { + app.insert_non_send_resource(MicrophoneEncoder( + Encoder::new(48_000, Channels::Mono, Application::Voip) + .expect("unable to create microphone audio compressing encoder"), + )); + } + app.insert_non_send_resource(MicrophoneDecoder { + channels_1_decoder: Decoder::new(48_000, Channels::Mono) + .expect("unable to create microphone audio compressing decoder"), + channels_2_decoder: Decoder::new(48_000, Channels::Stereo) .expect("unable to create microphone audio compressing decoder"), - )); + }); app.add_systems(Update, send_voice_msg); app.add_systems(Update, rec_voice_msg); app.add_systems(Update, bad_jitter_buffer); @@ -29,7 +41,10 @@ impl bevy::prelude::Plugin for VoiceChatPlugin { #[derive(Resource)] pub struct MicrophoneEncoder(pub Encoder); #[derive(Resource)] -pub struct MicrophoneDecoder(pub Decoder); +pub struct MicrophoneDecoder { + pub channels_1_decoder: Decoder, + pub channels_2_decoder: Decoder, +} unsafe impl Sync for MicrophoneEncoder {} unsafe impl Sync for MicrophoneDecoder {} @@ -39,18 +54,24 @@ fn send_voice_msg( mut event_writer: EventWriter, mut local_size: Local>, ) { + let mut channels = 1; + #[cfg(target_os = "android")] + { + channels = 2; + } while let Ok(mut audio) = microphone.0.lock().unwrap().try_recv() { local_size.append(&mut audio); } - if local_size.len() < 2880 { + if local_size.len() < 2880 * channels { return; } - while local_size.len() > 2880 { + while local_size.len() > 2880 * channels { event_writer.send(ClientToServerVoiceMsg( encoder .0 - .encode_vec_float(local_size.drain(0..2880).as_ref(), 2880) + .encode_vec_float(local_size.drain(0..(2880 * channels)).as_ref(), 2880 * channels) .expect("couldnt' encode audio"), + social_networking::client::Channels(channels as u16), )) } } @@ -63,18 +84,46 @@ fn rec_voice_msg( for event in event_reader.read() { let id2 = event.0; let audio = &event.1; + let channels = event.2; for (id, audio_sink) in players.iter_mut() { if id2 != id.0 { continue; } - let mut output = [0.0; 2880]; - microphone_decoder - .0 - .decode_float(audio, &mut output, false) - .expect("unable to decode audio"); - audio_sink - .sink - .append(rodio::buffer::SamplesBuffer::new(1, 48_000, output)); + let mut output1 = [0.0; 2880]; + let mut output2 = [0.0; 2880 * 2]; + + match channels { + social_networking::client::Channels(channels) => { + match channels { + 1 => { + microphone_decoder + .channels_1_decoder + .decode_float(audio, &mut output1, false) + .expect("unable to decode audio"); + } + 2 => { + microphone_decoder + .channels_2_decoder + .decode_float(audio, &mut output2, false) + .expect("unable to decode audio"); + } + _ => panic!("wrong number of audio channels for decoding"), + } + } + }; + match channels.0 { + 1 => { + audio_sink + .sink + .append(rodio::buffer::SamplesBuffer::new(channels.0, 48_000, output1)); + } + 2 => { + audio_sink + .sink + .append(rodio::buffer::SamplesBuffer::new(channels.0, 48_000, output2)); + } + _ => panic!("impossible") + } audio_sink.sink.play(); audio_sink.sink.set_volume(1.0); } diff --git a/apps/social/networking/src/client.rs b/apps/social/networking/src/client.rs index ce5c4ae..f9ded87 100644 --- a/apps/social/networking/src/client.rs +++ b/apps/social/networking/src/client.rs @@ -19,6 +19,7 @@ use lightyear::prelude::{ ClientId, Io, IoConfig, LinkConditionerConfig, NetworkTarget, PingConfig, Replicate, TransportConfig, }; +use serde::{Deserialize, Serialize}; use crate::data_model::Local; use crate::lightyear::{MyProtocol, ServerToClientAudioMsg}; @@ -122,10 +123,14 @@ impl Plugin for ClientVoiceChat { } } #[derive(Event)] -pub struct ClientToServerVoiceMsg(pub Vec); +pub struct ClientToServerVoiceMsg(pub Vec, pub Channels); + +#[derive(Clone, Copy, PartialEq, Debug, Serialize, +Deserialize)] +pub struct Channels(pub u16); #[derive(Event)] -pub struct ServerToClientVoiceMsg(pub ClientId, pub Vec); +pub struct ServerToClientVoiceMsg(pub ClientId, pub Vec, pub Channels); fn send_client_to_server_voice_msg( mut client: ResMut>, @@ -134,7 +139,7 @@ fn send_client_to_server_voice_msg( for audio_msg in event_reader.read() { client .send_message::( - crate::lightyear::ClientToServerAudioMsg(audio_msg.0.clone()), + crate::lightyear::ClientToServerAudioMsg(audio_msg.0.clone(), audio_msg.1.clone()), ) .expect("unable to send message"); } @@ -148,6 +153,6 @@ fn rec_server_voice_msgs( let msg = msg.message(); let client_id = msg.0; let audio = msg.1.clone(); - event_writer.send(ServerToClientVoiceMsg(client_id, audio)); + event_writer.send(ServerToClientVoiceMsg(client_id, audio, msg.2.clone())); } } diff --git a/apps/social/networking/src/lightyear.rs b/apps/social/networking/src/lightyear.rs index 7ef12b3..482cebc 100644 --- a/apps/social/networking/src/lightyear.rs +++ b/apps/social/networking/src/lightyear.rs @@ -26,10 +26,10 @@ protocolize! { } #[derive(Message, Serialize, Deserialize, Clone, Debug, PartialEq)] -pub struct ClientToServerAudioMsg(pub Vec); +pub struct ClientToServerAudioMsg(pub Vec, pub Channels); #[derive(Message, Serialize, Deserialize, Clone, Debug, PartialEq)] -pub struct ServerToClientAudioMsg(pub ClientId, pub Vec); +pub struct ServerToClientAudioMsg(pub ClientId, pub Vec, pub Channels); #[message_protocol(protocol = "MyProtocol")] pub enum Messages { @@ -93,3 +93,4 @@ pub fn shared_config() -> SharedConfig { pub use lightyear::prelude::client::Interpolated; use lightyear::prelude::Message; +use crate::client::Channels; diff --git a/apps/social/networking/src/server.rs b/apps/social/networking/src/server.rs index bd9f719..d2b104a 100644 --- a/apps/social/networking/src/server.rs +++ b/apps/social/networking/src/server.rs @@ -112,6 +112,7 @@ fn re_broadcast_audio( for message in messages.read() { let id2 = *message.context(); let audio = message.message().clone().0; + let channels = message.message().1; for id in server.client_ids().collect::>() { if id == id2 { continue; @@ -119,7 +120,7 @@ fn re_broadcast_audio( server .send_message::( id, - ServerToClientAudioMsg(id2, audio.clone()), + ServerToClientAudioMsg(id2, audio.clone(), channels), ) .unwrap(); }