Malek/fix audio on android (#60)

merging this so i can test the server easily

apps/social/client/src/custom_audio/microphone.rs

@@ -5,6 +5,7 @@ use bevy::prelude::{warn, Commands, Resource, Startup};
 use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
 use std::sync::mpsc::{channel, Receiver};
 use std::sync::Mutex;
+use bevy::prelude::info;
 
 pub struct MicrophonePlugin;
 
@@ -33,7 +34,7 @@ impl Default for MicrophoneConfig {
 }
 
 pub fn create_microphone(mut commands: Commands) {
-	let microphone_config = MicrophoneConfig::default();
+	let mut microphone_config = MicrophoneConfig::default();
 
 	// we wanna share the output from our thread loop thing in here continuously with the rest of bevy.
 	let (tx, rx) = channel();
@@ -49,11 +50,24 @@ pub fn create_microphone(mut commands: Commands) {
 			Ok(configs) => configs,
 			Err(err) => return warn!("supported stream config error, microphone functionality will be disabled, error: {}", err),
 		};
+		for config in configs {
+			warn!("supported microphone config: {:#?}", config);
+		}
+		let mut configs = match device.supported_input_configs() {
+			Ok(configs) => configs,
+			Err(err) => return warn!("supported stream config error, microphone functionality will be disabled, error: {}", err),
+		};
+
+		#[cfg(target_os = "android")]
+		{
+			microphone_config.channels = 2;
+		}
+
 		let config = match configs
             .find(|c| {
                 c.sample_format() == cpal::SampleFormat::F32 && c.channels() == microphone_config.channels
-                 && c.min_sample_rate().0 < microphone_config.sample_rate
-                 && c.max_sample_rate().0 > microphone_config.sample_rate
+                 && c.min_sample_rate().0 <= microphone_config.sample_rate
+                 && c.max_sample_rate().0 >= microphone_config.sample_rate
             })
         {
             None => return warn!("microphone config of {:?} not supported, microphone functionality will be disabled", microphone_config),

apps/social/client/src/voice_chat/mod.rs

@@ -12,14 +12,26 @@ pub struct VoiceChatPlugin;
 
 impl bevy::prelude::Plugin for VoiceChatPlugin {
 	fn build(&self, app: &mut App) {
-		app.insert_non_send_resource(MicrophoneEncoder(
-			Encoder::new(48_000, Channels::Mono, Application::Voip)
-				.expect("unable to create microphone audio compressing encoder"),
-		));
-		app.insert_non_send_resource(MicrophoneDecoder(
-			Decoder::new(48_000, Channels::Mono)
+		#[cfg(target_os = "android")]
+		{
+			app.insert_non_send_resource(crate::voice_chat::MicrophoneEncoder(
+				Encoder::new(48_000, Channels::Stereo, Application::Voip)
+					.expect("unable to create microphone audio compressing encoder"),
+			));
+		}
+		#[cfg(not(target_os = "android"))]
+		{
+			app.insert_non_send_resource(MicrophoneEncoder(
+				Encoder::new(48_000, Channels::Mono, Application::Voip)
+					.expect("unable to create microphone audio compressing encoder"),
+			));
+		}
+		app.insert_non_send_resource(MicrophoneDecoder {
+			channels_1_decoder: Decoder::new(48_000, Channels::Mono)
+					.expect("unable to create microphone audio compressing decoder"),
+			channels_2_decoder: Decoder::new(48_000, Channels::Stereo)
 				.expect("unable to create microphone audio compressing decoder"),
-		));
+		});
 		app.add_systems(Update, send_voice_msg);
 		app.add_systems(Update, rec_voice_msg);
 		app.add_systems(Update, bad_jitter_buffer);
@@ -29,7 +41,10 @@ impl bevy::prelude::Plugin for VoiceChatPlugin {
 #[derive(Resource)]
 pub struct MicrophoneEncoder(pub Encoder);
 #[derive(Resource)]
-pub struct MicrophoneDecoder(pub Decoder);
+pub struct MicrophoneDecoder {
+	pub channels_1_decoder: Decoder,
+	pub channels_2_decoder: Decoder,
+}
 unsafe impl Sync for MicrophoneEncoder {}
 unsafe impl Sync for MicrophoneDecoder {}
 
@@ -39,18 +54,24 @@ fn send_voice_msg(
 	mut event_writer: EventWriter<ClientToServerVoiceMsg>,
 	mut local_size: Local<Vec<f32>>,
 ) {
+	let mut channels = 1;
+	#[cfg(target_os = "android")]
+	{
+		channels = 2;
+	}
 	while let Ok(mut audio) = microphone.0.lock().unwrap().try_recv() {
 		local_size.append(&mut audio);
 	}
-	if local_size.len() < 2880 {
+	if local_size.len() < 2880 * channels {
 		return;
 	}
-	while local_size.len() > 2880 {
+	while local_size.len() > 2880 * channels {
 		event_writer.send(ClientToServerVoiceMsg(
 			encoder
 				.0
-				.encode_vec_float(local_size.drain(0..2880).as_ref(), 2880)
+				.encode_vec_float(local_size.drain(0..(2880 * channels)).as_ref(), 2880 * channels)
 				.expect("couldnt' encode audio"),
+			social_networking::client::Channels(channels as u16),
 		))
 	}
 }
@@ -63,18 +84,46 @@ fn rec_voice_msg(
 	for event in event_reader.read() {
 		let id2 = event.0;
 		let audio = &event.1;
+		let channels = event.2;
 		for (id, audio_sink) in players.iter_mut() {
 			if id2 != id.0 {
 				continue;
 			}
-			let mut output = [0.0; 2880];
-			microphone_decoder
-				.0
-				.decode_float(audio, &mut output, false)
-				.expect("unable to decode audio");
-			audio_sink
-				.sink
-				.append(rodio::buffer::SamplesBuffer::new(1, 48_000, output));
+			let mut output1 = [0.0; 2880];
+			let mut output2 = [0.0; 2880 * 2];
+
+			match channels {
+				social_networking::client::Channels(channels) => {
+					match channels {
+						1 => {
+							microphone_decoder
+								.channels_1_decoder
+								.decode_float(audio, &mut output1, false)
+								.expect("unable to decode audio");
+						}
+						2 => {
+							microphone_decoder
+								.channels_2_decoder
+								.decode_float(audio, &mut output2, false)
+								.expect("unable to decode audio");
+						}
+						_ => panic!("wrong number of audio channels for decoding"),
+					}
+				}
+			};
+			match channels.0 {
+				1 => {
+					audio_sink
+						.sink
+						.append(rodio::buffer::SamplesBuffer::new(channels.0, 48_000, output1));
+				}
+				2 => {
+					audio_sink
+						.sink
+						.append(rodio::buffer::SamplesBuffer::new(channels.0, 48_000, output2));
+				}
+				_ => panic!("impossible")
+			}
 			audio_sink.sink.play();
 			audio_sink.sink.set_volume(1.0);
 		}

apps/social/networking/src/client.rs

@@ -19,6 +19,7 @@ use lightyear::prelude::{
 	ClientId, Io, IoConfig, LinkConditionerConfig, NetworkTarget, PingConfig,
 	Replicate, TransportConfig,
 };
+use serde::{Deserialize, Serialize};
 
 use crate::data_model::Local;
 use crate::lightyear::{MyProtocol, ServerToClientAudioMsg};
@@ -122,10 +123,14 @@ impl Plugin for ClientVoiceChat {
 	}
 }
 #[derive(Event)]
-pub struct ClientToServerVoiceMsg(pub Vec<u8>);
+pub struct ClientToServerVoiceMsg(pub Vec<u8>, pub Channels);
+
+#[derive(Clone, Copy, PartialEq, Debug, Serialize,
+Deserialize)]
+pub struct Channels(pub u16);
 
 #[derive(Event)]
-pub struct ServerToClientVoiceMsg(pub ClientId, pub Vec<u8>);
+pub struct ServerToClientVoiceMsg(pub ClientId, pub Vec<u8>, pub Channels);
 
 fn send_client_to_server_voice_msg(
 	mut client: ResMut<lightyear::client::resource::Client<MyProtocol>>,
@@ -134,7 +139,7 @@ fn send_client_to_server_voice_msg(
 	for audio_msg in event_reader.read() {
 		client
 			.send_message::<crate::lightyear::AudioChannel, _>(
-				crate::lightyear::ClientToServerAudioMsg(audio_msg.0.clone()),
+				crate::lightyear::ClientToServerAudioMsg(audio_msg.0.clone(), audio_msg.1.clone()),
 			)
 			.expect("unable to send message");
 	}
@@ -148,6 +153,6 @@ fn rec_server_voice_msgs(
 		let msg = msg.message();
 		let client_id = msg.0;
 		let audio = msg.1.clone();
-		event_writer.send(ServerToClientVoiceMsg(client_id, audio));
+		event_writer.send(ServerToClientVoiceMsg(client_id, audio, msg.2.clone()));
 	}
 }

apps/social/networking/src/lightyear.rs

@@ -26,10 +26,10 @@ protocolize! {
 }
 
 #[derive(Message, Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct ClientToServerAudioMsg(pub Vec<u8>);
+pub struct ClientToServerAudioMsg(pub Vec<u8>, pub Channels);
 
 #[derive(Message, Serialize, Deserialize, Clone, Debug, PartialEq)]
-pub struct ServerToClientAudioMsg(pub ClientId, pub Vec<u8>);
+pub struct ServerToClientAudioMsg(pub ClientId, pub Vec<u8>, pub Channels);
 
 #[message_protocol(protocol = "MyProtocol")]
 pub enum Messages {
@@ -93,3 +93,4 @@ pub fn shared_config() -> SharedConfig {
 
 pub use lightyear::prelude::client::Interpolated;
 use lightyear::prelude::Message;
+use crate::client::Channels;

apps/social/networking/src/server.rs

@@ -112,14 +112,15 @@ fn re_broadcast_audio(
 	for message in messages.read() {
 		let id2 = *message.context();
 		let audio = message.message().clone().0;
+		let channels = message.message().1;
 		for id in server.client_ids().collect::<Vec<_>>() {
 			if id == id2 {
 				continue;
 			}
 			server
 				.send_message::<AudioChannel, _>(
 					id,
-					ServerToClientAudioMsg(id2, audio.clone()),
+					ServerToClientAudioMsg(id2, audio.clone(), channels),
 				)
 				.unwrap();
 		}