Skip to content

Commit

Permalink
Merge pull request #3598 from vector-im/feature/ons/voice_message
Browse files Browse the repository at this point in the history
Voice Message
  • Loading branch information
onurays committed Jul 30, 2021
2 parents 8e28872 + 1aa706d commit c6bd6e4
Show file tree
Hide file tree
Showing 74 changed files with 2,607 additions and 57 deletions.
3 changes: 3 additions & 0 deletions build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,9 @@ allprojects {
// Chat effects
includeGroupByRegex 'com\\.github\\.jetradarmobile'
includeGroupByRegex 'nl\\.dionsegijn'

// Voice RecordView
includeGroupByRegex 'com\\.github\\.Armen101'
}
}
maven { url 'https://oss.sonatype.org/content/repositories/snapshots/' }
Expand Down
2 changes: 2 additions & 0 deletions library/ui-styles/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -60,4 +60,6 @@ dependencies {
implementation 'com.github.vector-im:PFLockScreen-Android:1.0.0-beta12'
// dialpad dimen
implementation 'im.dlg:android-dialer:1.2.5'
// AudioRecordView attr
implementation 'com.github.Armen101:AudioRecordView:1.0.5'
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<shape xmlns:android="http://schemas.android.com/apk/res/android">

<solid android:color="#F00" />

<corners android:radius="8dp" />

</shape>
1 change: 1 addition & 0 deletions library/ui-styles/src/main/res/values-ldrtl/integers.xml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>

<integer name="rtl_x_multiplier">-1</integer>
<integer name="rtl_mirror_flip">180</integer>

</resources>
4 changes: 4 additions & 0 deletions library/ui-styles/src/main/res/values/colors.xml
Original file line number Diff line number Diff line change
Expand Up @@ -128,4 +128,8 @@
<color name="vctr_chat_effect_snow_background_light">@color/black_alpha</color>
<color name="vctr_chat_effect_snow_background_dark">@android:color/transparent</color>

<attr name="vctr_voice_message_toast_background" format="color" />
<color name="vctr_voice_message_toast_background_light">@color/palette_black_900</color>
<color name="vctr_voice_message_toast_background_dark">@color/palette_gray_400</color>

</resources>
1 change: 1 addition & 0 deletions library/ui-styles/src/main/res/values/integers.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@

<integer name="default_animation_offset">200</integer>

<integer name="rtl_x_multiplier">1</integer>
<integer name="rtl_mirror_flip">0</integer>

<integer name="splash_animation_velocity">750</integer>
Expand Down
2 changes: 1 addition & 1 deletion library/ui-styles/src/main/res/values/palette.xml
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

<!-- For light themes -->
<color name="palette_gray_25">#F4F6FA</color>
<color name="palette_gray_50">#E6E8F0</color>
<color name="palette_gray_50">#E3E8F0</color>
<color name="palette_gray_100">#C1C6CD</color>
<color name="palette_gray_150">#8D97A5</color>
<color name="palette_gray_200">#737D8C</color>
Expand Down
26 changes: 26 additions & 0 deletions library/ui-styles/src/main/res/values/styles_voice_message.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="utf-8"?>
<resources>

<style name="VoicePlaybackWaveform">
<item name="chunkColor">?vctr_content_secondary</item>
<item name="chunkAlignTo">center</item>
<item name="chunkMinHeight">1dp</item>
<item name="chunkRoundedCorners">true</item>
<item name="chunkSoftTransition">true</item>
<item name="chunkSpace">2dp</item>
<item name="chunkWidth">2dp</item>
<item name="direction">rightToLeft</item>
</style>

<style name="Widget.Vector.TextView.Caption.Toast">
<item name="android:paddingTop">8dp</item>
<item name="android:paddingBottom">8dp</item>
<item name="android:paddingStart">12dp</item>
<item name="android:paddingEnd">12dp</item>
<item name="android:background">@drawable/bg_round_corner_8dp</item>
<item name="android:backgroundTint">?vctr_voice_message_toast_background</item>
<item name="android:textColor">@color/palette_white</item>
<item name="android:gravity">center</item>
</style>

</resources>
2 changes: 2 additions & 0 deletions library/ui-styles/src/main/res/values/theme_dark.xml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,8 @@

<item name="vctr_jump_to_unread_style">@style/Widget.Vector.JumpToUnread.Dark</item>

<!-- Voice Message -->
<item name="vctr_voice_message_toast_background">@color/vctr_voice_message_toast_background_dark</item>
</style>

<style name="Theme.Vector.Dark" parent="Base.Theme.Vector.Dark" />
Expand Down
2 changes: 2 additions & 0 deletions library/ui-styles/src/main/res/values/theme_light.xml
Original file line number Diff line number Diff line change
Expand Up @@ -137,6 +137,8 @@

<item name="vctr_jump_to_unread_style">@style/Widget.Vector.JumpToUnread.Light</item>

<!-- Voice Message -->
<item name="vctr_voice_message_toast_background">@color/vctr_voice_message_toast_background_light</item>
</style>

<style name="Theme.Vector.Light" parent="Base.Theme.Vector.Light" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import io.reactivex.Single
import kotlinx.coroutines.rx2.rxCompletable
import kotlinx.coroutines.rx2.rxSingle
import org.matrix.android.sdk.api.query.QueryStringValue
import org.matrix.android.sdk.api.session.content.ContentAttachmentData
import org.matrix.android.sdk.api.session.events.model.Event
import org.matrix.android.sdk.api.session.identity.ThreePid
import org.matrix.android.sdk.api.session.room.Room
Expand Down Expand Up @@ -146,6 +147,10 @@ class RxRoom(private val room: Room) {
fun deleteAvatar(): Completable = rxCompletable {
room.deleteAvatar()
}

fun sendMedia(attachment: ContentAttachmentData, compressBeforeSending: Boolean, roomIds: Set<String>): Completable = rxCompletable {
room.sendMedia(attachment, compressBeforeSending, roomIds)
}
}

fun Room.rx(): RxRoom {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ data class ContentAttachmentData(
val name: String? = null,
val queryUri: Uri,
val mimeType: String?,
val type: Type
val type: Type,
val waveform: List<Int>? = null
) : Parcelable {

@JsonClass(generateAdapter = false)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,15 +24,15 @@ data class AudioInfo(
/**
* The mimetype of the audio e.g. "audio/aac".
*/
@Json(name = "mimetype") val mimeType: String?,
@Json(name = "mimetype") val mimeType: String? = null,

/**
* The size of the audio clip in bytes.
*/
@Json(name = "size") val size: Long = 0,
@Json(name = "size") val size: Long? = null,

/**
* The duration of the audio in milliseconds.
*/
@Json(name = "duration") val duration: Int = 0
@Json(name = "duration") val duration: Int? = null
)
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright 2020 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.matrix.android.sdk.api.session.room.model.message

import com.squareup.moshi.Json
import com.squareup.moshi.JsonClass

/**
* See https://github.com/matrix-org/matrix-doc/blob/travis/msc/audio-waveform/proposals/3246-audio-waveform.md
*/
@JsonClass(generateAdapter = true)
data class AudioWaveformInfo(
@Json(name = "duration")
val duration: Int? = null,

/**
* The array should have no less than 30 elements and no more than 120.
* List of integers between zero and 1024, inclusive.
*/
@Json(name = "waveform")
val waveform: List<Int>? = null
)
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import com.squareup.moshi.Json
import com.squareup.moshi.JsonClass
import org.matrix.android.sdk.api.session.events.model.Content
import org.matrix.android.sdk.api.session.room.model.relation.RelationDefaultContent
import org.matrix.android.sdk.api.util.JsonDict
import org.matrix.android.sdk.internal.crypto.model.rest.EncryptedFileInfo

@JsonClass(generateAdapter = true)
Expand Down Expand Up @@ -50,7 +51,17 @@ data class MessageAudioContent(
/**
* Required if the file is encrypted. Information on the encrypted file, as specified in End-to-end encryption.
*/
@Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null
@Json(name = "file") override val encryptedFileInfo: EncryptedFileInfo? = null,

/**
* Encapsulates waveform and duration of the audio.
*/
@Json(name = "org.matrix.msc1767.audio") val audioWaveformInfo: AudioWaveformInfo? = null,

/**
* Indicates that is a voice message.
*/
@Json(name = "org.matrix.msc3245.voice") val voiceMessageIndicator: JsonDict? = null
) : MessageWithAttachmentContent {

override val mimeType: String?
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ object MimeTypes {
const val Jpeg = "image/jpeg"
const val Gif = "image/gif"

const val Ogg = "audio/ogg"

fun String?.normalizeMimeType() = if (this == BadJpg) Jpeg else this

fun String?.isMimeTypeImage() = this?.startsWith("image/").orFalse()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import kotlinx.coroutines.completeWith
import kotlinx.coroutines.withContext
import okhttp3.OkHttpClient
import okhttp3.Request
import org.matrix.android.sdk.api.failure.Failure
import org.matrix.android.sdk.api.session.content.ContentUrlResolver
import org.matrix.android.sdk.api.session.file.FileService
import org.matrix.android.sdk.internal.crypto.attachments.ElementToDecrypt
Expand Down Expand Up @@ -124,13 +125,21 @@ internal class DefaultFileService @Inject constructor(
.header(DOWNLOAD_PROGRESS_INTERCEPTOR_HEADER, url)
.build()

val response = okHttpClient.newCall(request).execute()
val response = try {
okHttpClient.newCall(request).execute()
} catch (failure: Throwable) {
throw if (failure is IOException) {
Failure.NetworkConnection(failure)
} else {
failure
}
}

if (!response.isSuccessful) {
throw IOException()
throw Failure.NetworkConnection(IOException())
}

val source = response.body?.source() ?: throw IOException()
val source = response.body?.source() ?: throw Failure.NetworkConnection(IOException())

Timber.v("Response size ${response.body?.contentLength()} - Stream available: ${!source.exhausted()}")

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ internal class DefaultSendService @AssistedInject constructor(
mimeType = messageContent.mimeType,
name = messageContent.body,
queryUri = Uri.parse(messageContent.url),
type = ContentAttachmentData.Type.AUDIO
type = ContentAttachmentData.Type.AUDIO,
waveform = messageContent.audioWaveformInfo?.waveform
)
localEchoRepository.updateSendState(localEcho.eventId, roomId, SendState.UNSENT)
internalSendMedia(listOf(localEcho.root), attachmentData, true)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import org.matrix.android.sdk.api.session.events.model.RelationType
import org.matrix.android.sdk.api.session.events.model.UnsignedData
import org.matrix.android.sdk.api.session.events.model.toContent
import org.matrix.android.sdk.api.session.room.model.message.AudioInfo
import org.matrix.android.sdk.api.session.room.model.message.AudioWaveformInfo
import org.matrix.android.sdk.api.session.room.model.message.FileInfo
import org.matrix.android.sdk.api.session.room.model.message.ImageInfo
import org.matrix.android.sdk.api.session.room.model.message.MessageAudioContent
Expand Down Expand Up @@ -74,6 +75,7 @@ internal class LocalEchoEventFactory @Inject constructor(
private val markdownParser: MarkdownParser,
private val textPillsUtils: TextPillsUtils,
private val thumbnailExtractor: ThumbnailExtractor,
private val waveformSanitizer: WaveFormSanitizer,
private val localEchoRepository: LocalEchoRepository,
private val permalinkFactory: PermalinkFactory
) {
Expand Down Expand Up @@ -289,14 +291,21 @@ internal class LocalEchoEventFactory @Inject constructor(
}

private fun createAudioEvent(roomId: String, attachment: ContentAttachmentData): Event {
val isVoiceMessage = attachment.waveform != null
val content = MessageAudioContent(
msgType = MessageType.MSGTYPE_AUDIO,
body = attachment.name ?: "audio",
audioInfo = AudioInfo(
duration = attachment.duration?.toInt(),
mimeType = attachment.getSafeMimeType()?.takeIf { it.isNotBlank() },
size = attachment.size
),
url = attachment.queryUri.toString()
url = attachment.queryUri.toString(),
audioWaveformInfo = if (!isVoiceMessage) null else AudioWaveformInfo(
duration = attachment.duration?.toInt(),
waveform = waveformSanitizer.sanitize(attachment.waveform)
),
voiceMessageIndicator = if (!isVoiceMessage) null else emptyMap()
)
return createMessageEvent(roomId, content)
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
/*
* Copyright (c) 2021 The Matrix.org Foundation C.I.C.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.matrix.android.sdk.internal.session.room.send

import timber.log.Timber
import javax.inject.Inject
import kotlin.math.abs
import kotlin.math.ceil

internal class WaveFormSanitizer @Inject constructor() {
private companion object {
const val MIN_NUMBER_OF_VALUES = 30
const val MAX_NUMBER_OF_VALUES = 120

const val MAX_VALUE = 1024
}

/**
* The array should have no less than 30 elements and no more than 120.
* List of integers between zero and 1024, inclusive.
*/
fun sanitize(waveForm: List<Int>?): List<Int>? {
if (waveForm.isNullOrEmpty()) {
return null
}

// Limit the number of items
val sizeInRangeList = mutableListOf<Int>()
when {
waveForm.size < MIN_NUMBER_OF_VALUES -> {
// Repeat the same value to have at least 30 items
val repeatTimes = ceil(MIN_NUMBER_OF_VALUES / waveForm.size.toDouble()).toInt()
waveForm.map { value ->
repeat(repeatTimes) {
sizeInRangeList.add(value)
}
}
}
waveForm.size > MAX_NUMBER_OF_VALUES -> {
val keepOneOf = ceil(waveForm.size.toDouble() / MAX_NUMBER_OF_VALUES).toInt()
waveForm.mapIndexed { idx, value ->
if (idx % keepOneOf == 0) {
sizeInRangeList.add(value)
}
}
}
else -> {
sizeInRangeList.addAll(waveForm)
}
}

// OK, ensure all items are positive
val positiveList = sizeInRangeList.map {
abs(it)
}

// Ensure max is not above MAX_VALUE
val max = positiveList.maxOrNull() ?: MAX_VALUE

val finalList = if (max > MAX_VALUE) {
// Reduce the values
positiveList.map {
it * MAX_VALUE / max
}
} else {
positiveList
}

Timber.d("Sanitize from ${waveForm.size} items to ${finalList.size} items. Max value was $max")
return finalList
}
}
Loading

0 comments on commit c6bd6e4

Please sign in to comment.